first commit

k3s/README.md

@@ -0,0 +1,7 @@
+- if you reinstall agent, you must copy password from master and paste it to /var/lib/rancher/k3s/agent/node-password.txt. the password is from /var/lib/rancher/k3s/server/cred/passwd in master server
+
+- if you want to run "sudo kubectl get nodes" in agent, you must copy k3s.yaml(in /etc/rancher/k3s) from master and paste it to /etc/rancher/k3s/ in master
+
+- K3S_TOKEN is the content of /var/lib/rancher/k3s/server/node-token in master
+
+- if you set the nodename of agent, you must add ip nodename(such as 127.0.0.1   node-1) to /etc/hosts

k3s/agent-install.sh

@@ -0,0 +1,16 @@
+sudo cp ./k3s /usr/local/bin/
+
+sudo apt-get update;
+sudo apt-get install nfs-common
+
+read -p "input master token: " input;
+token=`echo $input`;
+read -p "input master endpoint: " input;
+master=`echo $input | tr '[A-Z]' '[a-z]'`;
+read -p "input node name: " input;
+nodename=`echo $input | tr '[A-Z]' '[a-z]'`;
+
+sudo mkdir -p /var/lib/rancher/k3s/agent/images/
+sudo cp ./k3s-airgap-images-amd64.tar /var/lib/rancher/k3s/agent/images/
+
+sudo INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=$token K3S_URL=$master K3S_NODE_NAME=$nodename ./install.sh

k3s/install.sh

@@ -0,0 +1,646 @@
+#!/bin/sh
+set -e
+
+# Usage:
+#   curl ... | ENV_VAR=... sh -
+#       or
+#   ENV_VAR=... ./install.sh
+#
+# Example:
+#   Installing a server without an agent:
+#     curl ... | INSTALL_K3S_EXEC="--disable-agent" sh -
+#   Installing an agent to point at a server:
+#     curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh -
+#
+# Environment variables:
+#   - K3S_*
+#     Environment variables which begin with K3S_ will be preserved for the
+#     systemd service to use. Setting K3S_URL without explicitly setting
+#     a systemd exec command will default the command to "agent", and we
+#     enforce that K3S_TOKEN or K3S_CLUSTER_SECRET is also set.
+#
+#   - INSTALL_K3S_SKIP_DOWNLOAD
+#     If set to true will not download k3s hash or binary.
+#
+#   - INSTALL_K3S_SKIP_START
+#     If set to true will not start k3s service.
+#
+#   - INSTALL_K3S_VERSION
+#     Version of k3s to download from github. Will attempt to download the
+#     latest version if not specified.
+#
+#   - INSTALL_K3S_BIN_DIR
+#     Directory to install k3s binary, links, and uninstall script to, or use
+#     /usr/local/bin as the default
+#
+#   - INSTALL_K3S_BIN_DIR_READ_ONLY
+#     If set to true will not write files to INSTALL_K3S_BIN_DIR, forces
+#     setting INSTALL_K3S_SKIP_DOWNLOAD=true
+#
+#   - INSTALL_K3S_SYSTEMD_DIR
+#     Directory to install systemd service and environment files to, or use
+#     /etc/systemd/system as the default
+#
+#   - INSTALL_K3S_EXEC or script arguments
+#     Command with flags to use for launching k3s in the systemd service, if
+#     the command is not specified will default to "agent" if K3S_URL is set
+#     or "server" if not. The final systemd command resolves to a combination
+#     of EXEC and script args ($@).
+#
+#     The following commands result in the same behavior:
+#       curl ... | INSTALL_K3S_EXEC="--disable-agent" sh -s -
+#       curl ... | INSTALL_K3S_EXEC="server --disable-agent" sh -s -
+#       curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable-agent
+#       curl ... | sh -s - server --disable-agent
+#       curl ... | sh -s - --disable-agent
+#
+#   - INSTALL_K3S_NAME
+#     Name of systemd service to create, will default from the k3s exec command
+#     if not specified. If specified the name will be prefixed with 'k3s-'.
+#
+#   - INSTALL_K3S_TYPE
+#     Type of systemd service to create, will default from the k3s exec command
+#     if not specified.
+
+GITHUB_URL=https://github.com/rancher/k3s/releases
+
+# --- helper functions for logs ---
+info()
+{
+    echo "[INFO] " "$@"
+}
+fatal()
+{
+    echo "[ERROR] " "$@"
+    exit 1
+}
+
+# --- fatal if no systemd or openrc ---
+verify_system() {
+    if [ -x /sbin/openrc-run ]; then
+        HAS_OPENRC=true
+        return
+    fi
+    if [ -d /run/systemd ]; then
+        HAS_SYSTEMD=true
+        return
+    fi
+    fatal "Can not find systemd or openrc to use as a process supervisor for k3s"
+}
+
+# --- add quotes to command arguments ---
+quote() {
+    for arg in "$@"; do
+        printf "%s\n" "$arg" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"
+    done
+}
+
+# --- add indentation and trailing slash to quoted args ---
+quote_indent() {
+    printf ' \\'"\n"
+    for arg in "$@"; do
+        printf "\t%s "'\\'"\n" "$(quote "$arg")"
+    done
+}
+
+# --- escape most punctuation characters, except quotes, forward slash, and space ---
+escape() {
+    printf "%s" "$@" | sed -e 's/\([][!#$%&()*;<=>?\_`{|}]\)/\\\1/g;'
+}
+
+# --- escape double quotes ---
+escape_dq() {
+    printf "%s" "$@" | sed -e 's/"/\\"/g'
+}
+
+# --- define needed environment variables ---
+setup_env() {
+    # --- use command args if passed or create default ---
+    case "$1" in
+        # --- if we only have flags discover if command should be server or agent ---
+        (-*|"")
+            if [ -z "${K3S_URL}" ]; then
+                CMD_K3S=server
+            else
+                if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_CLUSTER_SECRET}" ]; then
+                    fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN or K3S_CLUSTER_SECRET is not defined."
+                fi
+                CMD_K3S=agent
+            fi
+        ;;
+        # --- command is provided ---
+        (*)
+            CMD_K3S="$1"
+            shift
+        ;;
+    esac
+    CMD_K3S_EXEC="${CMD_K3S}$(quote_indent "$@")"
+
+    # --- use systemd name if defined or create default ---
+    if [ -n "${INSTALL_K3S_NAME}" ]; then
+        SYSTEM_NAME=k3s-${INSTALL_K3S_NAME}
+    else
+        if [ "${CMD_K3S}" = "server" ]; then
+            SYSTEM_NAME=k3s
+        else
+            SYSTEM_NAME=k3s-${CMD_K3S}
+        fi
+    fi
+
+    # --- check for invalid characters in system name ---
+    valid_chars=$(printf "%s" "${SYSTEM_NAME}" | sed -e 's/[][!#$%&()*;<=>?\_`{|}/[:space:]]/^/g;' )
+    if [ "${SYSTEM_NAME}" != "${valid_chars}"  ]; then
+        invalid_chars=$(printf "%s" "${valid_chars}" | sed -e 's/[^^]/ /g')
+        fatal "Invalid characters for system name:
+            ${SYSTEM_NAME}
+            ${invalid_chars}"
+    fi
+
+    # --- set related files from system name ---
+    SERVICE_K3S=${SYSTEM_NAME}.service
+    UNINSTALL_K3S_SH=${SYSTEM_NAME}-uninstall.sh
+    KILLALL_K3S_SH=k3s-killall.sh
+
+    # --- use sudo if we are not already root ---
+    SUDO=sudo
+    if [ `id -u` = 0 ]; then
+        SUDO=
+    fi
+
+    # --- use systemd type if defined or create default ---
+    if [ -n "${INSTALL_K3S_TYPE}" ]; then
+        SYSTEMD_TYPE="${INSTALL_K3S_TYPE}"
+    else
+        if [ "${CMD_K3S}" = "server" ]; then
+            SYSTEMD_TYPE=notify
+        else
+            SYSTEMD_TYPE=exec
+        fi
+    fi
+
+    # --- use binary install directory if defined or create default ---
+    if [ -n "${INSTALL_K3S_BIN_DIR}" ]; then
+        BIN_DIR="${INSTALL_K3S_BIN_DIR}"
+    else
+        BIN_DIR="/usr/local/bin"
+    fi
+
+    # --- use systemd directory if defined or create default ---
+    if [ -n "${INSTALL_K3S_SYSTEMD_DIR}" ]; then
+        SYSTEMD_DIR="${INSTALL_K3S_SYSTEMD_DIR}"
+    else
+        SYSTEMD_DIR="/etc/systemd/system"
+    fi
+
+    # --- use servive or environment location depending on systemd/openrc ---
+    if [ "${HAS_SYSTEMD}" = "true" ]; then
+        FILE_K3S_SERVICE=${SYSTEMD_DIR}/${SERVICE_K3S}
+        FILE_K3S_ENV=${SYSTEMD_DIR}/${SERVICE_K3S}.env
+    elif [ "${HAS_OPENRC}" = "true" ]; then
+        $SUDO mkdir -p /etc/rancher/k3s
+        FILE_K3S_SERVICE=/etc/init.d/${SYSTEM_NAME}
+        FILE_K3S_ENV=/etc/rancher/k3s/${SYSTEM_NAME}.env
+    fi
+
+    # --- get hash of config & exec for currently installed k3s ---
+    PRE_INSTALL_HASHES=`get_installed_hashes`
+
+    # --- if bin directory is read only skip download ---
+    if [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = "true" ]; then
+        INSTALL_K3S_SKIP_DOWNLOAD=true
+    fi
+}
+
+# --- check if skip download environment variable set ---
+can_skip_download() {
+    if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != "true" ]; then
+        return 1
+    fi
+}
+
+# --- verify an executabe k3s binary is installed ---
+verify_k3s_is_executable() {
+    if [ ! -x ${BIN_DIR}/k3s ]; then
+        fatal "Executable k3s binary not found at ${BIN_DIR}/k3s"
+    fi
+}
+
+# --- set arch and suffix, fatal if architecture not supported ---
+setup_verify_arch() {
+    if [ -z "$ARCH" ]; then
+        ARCH=`uname -m`
+    fi
+    case $ARCH in
+        amd64)
+            ARCH=amd64
+            SUFFIX=
+            ;;
+        x86_64)
+            ARCH=amd64
+            SUFFIX=
+            ;;
+        arm64)
+            ARCH=arm64
+            SUFFIX=-${ARCH}
+            ;;
+        aarch64)
+            ARCH=arm64
+            SUFFIX=-${ARCH}
+            ;;
+        arm*)
+            ARCH=arm
+            SUFFIX=-${ARCH}hf
+            ;;
+        *)
+            fatal "Unsupported architecture $ARCH"
+    esac
+}
+
+# --- fatal if no curl ---
+verify_curl() {
+    if [ -z `which curl || true` ]; then
+        fatal "Can not find curl for downloading files"
+    fi
+}
+
+# --- create tempory directory and cleanup when done ---
+setup_tmp() {
+    TMP_DIR=`mktemp -d -t k3s-install.XXXXXXXXXX`
+    TMP_HASH=${TMP_DIR}/k3s.hash
+    TMP_BIN=${TMP_DIR}/k3s.bin
+    cleanup() {
+        code=$?
+        set +e
+        trap - EXIT
+        rm -rf ${TMP_DIR}
+        exit $code
+    }
+    trap cleanup INT EXIT
+}
+
+# --- use desired k3s version if defined or find latest ---
+get_release_version() {
+    if [ -n "${INSTALL_K3S_VERSION}" ]; then
+        VERSION_K3S="${INSTALL_K3S_VERSION}"
+    else
+        info "Finding latest release"
+        VERSION_K3S=`curl -w "%{url_effective}" -I -L -s -S ${GITHUB_URL}/latest -o /dev/null | sed -e 's|.*/||'`
+    fi
+    info "Using ${VERSION_K3S} as release"
+}
+
+# --- download hash from github url ---
+download_hash() {
+    HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt
+    info "Downloading hash ${HASH_URL}"
+    curl -o ${TMP_HASH} -sfL ${HASH_URL} || fatal "Hash download failed"
+    HASH_EXPECTED=`grep " k3s${SUFFIX}$" ${TMP_HASH} | awk '{print $1}'`
+}
+
+# --- check hash against installed version ---
+installed_hash_matches() {
+    if [ -x ${BIN_DIR}/k3s ]; then
+        HASH_INSTALLED=`sha256sum ${BIN_DIR}/k3s | awk '{print $1}'`
+        if [ "${HASH_EXPECTED}" = "${HASH_INSTALLED}" ]; then
+            return
+        fi
+    fi
+    return 1
+}
+
+# --- download binary from github url ---
+download_binary() {
+    BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX}
+    info "Downloading binary ${BIN_URL}"
+    curl -o ${TMP_BIN} -sfL ${BIN_URL} || fatal "Binary download failed"
+}
+
+# --- verify downloaded binary hash ---
+verify_binary() {
+    info "Verifying binary download"
+    HASH_BIN=`sha256sum ${TMP_BIN} | awk '{print $1}'`
+    if [ "${HASH_EXPECTED}" != "${HASH_BIN}" ]; then
+        fatal "Download sha256 does not match ${HASH_EXPECTED}, got ${HASH_BIN}"
+    fi
+}
+
+# --- setup permissions and move binary to system directory ---
+setup_binary() {
+    chmod 755 ${TMP_BIN}
+    info "Installing k3s to ${BIN_DIR}/k3s"
+    $SUDO chown root:root ${TMP_BIN}
+    $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/k3s
+
+    if command -v getenforce > /dev/null 2>&1; then
+        if [ "Disabled" != `getenforce` ]; then
+            info "SeLinux is enabled, setting permissions"
+            if ! $SUDO semanage fcontext -l | grep "${BIN_DIR}/k3s" > /dev/null 2>&1; then
+                $SUDO semanage fcontext -a -t bin_t "${BIN_DIR}/k3s"
+            fi
+            $SUDO restorecon -v ${BIN_DIR}/k3s > /dev/null
+        fi
+    fi
+}
+
+# --- download and verify k3s ---
+download_and_verify() {
+    if can_skip_download; then
+       info "Skipping k3s download and verify"
+       verify_k3s_is_executable
+       return
+    fi
+
+    setup_verify_arch
+    verify_curl
+    setup_tmp
+    get_release_version
+    download_hash
+
+    if installed_hash_matches; then
+        info "Skipping binary downloaded, installed k3s matches hash"
+        return
+    fi
+
+    download_binary
+    verify_binary
+    setup_binary
+}
+
+# --- add additional utility links ---
+create_symlinks() {
+    [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = "true" ] && return
+    if [ ! -e ${BIN_DIR}/kubectl ]; then
+        info "Creating ${BIN_DIR}/kubectl symlink to k3s"
+        $SUDO ln -s k3s ${BIN_DIR}/kubectl
+    fi
+
+    if [ ! -e ${BIN_DIR}/crictl ]; then
+        info "Creating ${BIN_DIR}/crictl symlink to k3s"
+        $SUDO ln -s k3s ${BIN_DIR}/crictl
+    fi
+}
+
+
+# --- create killall script ---
+create_killall() {
+    [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = "true" ] && return
+    info "Creating killall script ${BIN_DIR}/${KILLALL_K3S_SH}"
+    $SUDO tee ${BIN_DIR}/${KILLALL_K3S_SH} >/dev/null << \EOF
+#!/bin/sh
+set -x
+[ `id -u` = 0 ] || exec sudo $0 $@
+
+for bin in /var/lib/rancher/k3s/data/**/bin/; do
+    [ -d $bin ] && export PATH=$bin:$PATH
+done
+
+for service in /etc/systemd/system/k3s*.service; do
+    [ -s $service ] && systemctl stop $(basename $service)
+done
+
+for service in /etc/init.d/k3s*; do
+    [ -x $service ] && $service stop
+done
+
+pstree() {
+    for pid in $@; do
+        echo $pid
+        pstree $(ps -o ppid= -o pid= | awk "\$1==$pid {print \$2}")
+    done
+}
+
+killtree() {
+    [ $# -ne 0 ] && kill $(set +x; pstree $@; set -x)
+}
+
+killtree $(lsof | sed -e 's/^[^0-9]*//g; s/  */\t/g' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1 | sort -n -u)
+
+do_unmount() {
+    MOUNTS=`cat /proc/self/mounts | awk '{print $2}' | grep "^$1" | sort -r`
+    if [ -n "${MOUNTS}" ]; then
+        umount ${MOUNTS}
+    fi
+}
+
+do_unmount '/run/k3s'
+do_unmount '/var/lib/rancher/k3s'
+
+nets=$(ip link show | grep 'master cni0' | awk -F': ' '{print $2}' | sed -e 's|@.*||')
+for iface in $nets; do
+    ip link delete $iface;
+done
+ip link delete cni0
+ip link delete flannel.1
+rm -rf /var/lib/cni/
+EOF
+    $SUDO chmod 755 ${BIN_DIR}/${KILLALL_K3S_SH}
+    $SUDO chown root:root ${BIN_DIR}/${KILLALL_K3S_SH}
+}
+
+# --- create uninstall script ---
+create_uninstall() {
+    [ "${INSTALL_K3S_BIN_DIR_READ_ONLY}" = "true" ] && return
+    info "Creating uninstall script ${BIN_DIR}/${UNINSTALL_K3S_SH}"
+    $SUDO tee ${BIN_DIR}/${UNINSTALL_K3S_SH} >/dev/null << EOF
+#!/bin/sh
+set -x
+[ \`id -u\` = 0 ] || exec sudo \$0 \$@
+
+${BIN_DIR}/${KILLALL_K3S_SH}
+
+if which systemctl; then
+    systemctl disable ${SYSTEM_NAME}
+    systemctl reset-failed ${SYSTEM_NAME}
+    systemctl daemon-reload
+fi
+if which rc-update; then
+    rc-update delete ${SYSTEM_NAME} default
+fi
+
+rm -f ${FILE_K3S_SERVICE}
+rm -f ${FILE_K3S_ENV}
+
+remove_uninstall() {
+    rm -f ${BIN_DIR}/${UNINSTALL_K3S_SH}
+}
+trap remove_uninstall EXIT
+
+if (ls ${SYSTEMD_DIR}/k3s*.service || ls /etc/init.d/k3s*) >/dev/null 2>&1; then
+    set +x; echo "Additional k3s services installed, skipping uninstall of k3s"; set -x
+    exit
+fi
+
+if [ -L ${BIN_DIR}/kubectl ]; then
+    rm -f ${BIN_DIR}/kubectl
+fi
+if [ -L ${BIN_DIR}/crictl ]; then
+    rm -f ${BIN_DIR}/crictl
+fi
+
+rm -rf /etc/rancher/k3s
+rm -rf /var/lib/rancher/k3s
+rm -f ${BIN_DIR}/k3s
+rm -f ${BIN_DIR}/${KILLALL_K3S_SH}
+EOF
+    $SUDO chmod 755 ${BIN_DIR}/${UNINSTALL_K3S_SH}
+    $SUDO chown root:root ${BIN_DIR}/${UNINSTALL_K3S_SH}
+}
+
+# --- disable current service if loaded --
+systemd_disable() {
+    $SUDO rm -f /etc/systemd/system/${SERVICE_K3S} || true
+    $SUDO rm -f /etc/systemd/system/${SERVICE_K3S}.env || true
+    $SUDO systemctl disable ${SYSTEM_NAME} >/dev/null 2>&1 || true
+}
+
+# --- capture current env and create file containing k3s_ variables ---
+create_env_file() {
+    info "env: Creating environment file ${FILE_K3S_ENV}"
+    UMASK=`umask`
+    umask 0377
+    env | grep '^K3S_' | $SUDO tee ${FILE_K3S_ENV} >/dev/null
+    umask $UMASK
+}
+
+# --- write systemd service file ---
+create_systemd_service_file() {
+    info "systemd: Creating service file ${FILE_K3S_SERVICE}"
+    $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+After=network-online.target
+
+[Service]
+Type=${SYSTEMD_TYPE}
+EnvironmentFile=${FILE_K3S_ENV}
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart=${BIN_DIR}/k3s \\
+    ${CMD_K3S_EXEC}
+
+KillMode=process
+Delegate=yes
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+EOF
+}
+
+# --- write openrc service file ---
+create_openrc_service_file() {
+    LOG_FILE=/var/log/${SYSTEM_NAME}.log
+
+    info "openrc: Creating service file ${FILE_K3S_SERVICE}"
+    $SUDO tee ${FILE_K3S_SERVICE} >/dev/null << EOF
+#!/sbin/openrc-run
+
+depend() {
+    after net-online
+    need net
+}
+
+start_pre() {
+    rm -f /tmp/k3s.*
+}
+
+supervisor=supervise-daemon
+name="${SYSTEM_NAME}"
+command="${BIN_DIR}/k3s"
+command_args="$(escape_dq "${CMD_K3S_EXEC}")
+    >>${LOG_FILE} 2>&1"
+
+pidfile="/var/run/${SYSTEM_NAME}.pid"
+respawn_delay=5
+
+set -o allexport
+if [ -f /etc/environment ]; then source /etc/environment; fi
+if [ -f ${FILE_K3S_ENV} ]; then source ${FILE_K3S_ENV}; fi
+set +o allexport
+EOF
+    $SUDO chmod 0755 ${FILE_K3S_SERVICE}
+
+    $SUDO tee /etc/logrotate.d/${SYSTEM_NAME} >/dev/null << EOF
+${LOG_FILE} {
+	missingok
+	notifempty
+	copytruncate
+}
+EOF
+}
+
+# --- write systemd or openrc service file ---
+create_service_file() {
+    [ "${HAS_SYSTEMD}" = "true" ] && create_systemd_service_file
+    [ "${HAS_OPENRC}" = "true" ] && create_openrc_service_file
+    return 0
+}
+
+# --- get hashes of the current k3s bin and service files
+get_installed_hashes() {
+    $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true
+}
+
+# --- enable and start systemd service ---
+systemd_enable() {
+    info "systemd: Enabling ${SYSTEM_NAME} unit"
+    $SUDO systemctl enable ${FILE_K3S_SERVICE} >/dev/null
+    $SUDO systemctl daemon-reload >/dev/null
+}
+
+systemd_start() {
+    info "systemd: Starting ${SYSTEM_NAME}"
+    $SUDO systemctl restart ${SYSTEM_NAME}
+}
+
+# --- enable and start openrc service ---
+openrc_enable() {
+    info "openrc: Enabling ${SYSTEM_NAME} service for default runlevel"
+    $SUDO rc-update add ${SYSTEM_NAME} default >/dev/null
+}
+
+openrc_start() {
+    info "openrc: Starting ${SYSTEM_NAME}"
+    $SUDO ${FILE_K3S_SERVICE} restart
+}
+
+# --- startup systemd or openrc service ---
+service_enable_and_start() {
+    [ "${HAS_SYSTEMD}" = "true" ] && systemd_enable
+    [ "${HAS_OPENRC}" = "true" ] && openrc_enable
+
+    [ "${INSTALL_K3S_SKIP_START}" = "true" ] && return
+
+    POST_INSTALL_HASHES=`get_installed_hashes`
+    if [ "${PRE_INSTALL_HASHES}" = "${POST_INSTALL_HASHES}" ]; then
+        info "No change detected so skipping service start"
+        return
+    fi
+
+    [ "${HAS_SYSTEMD}" = "true" ] && systemd_start
+    [ "${HAS_OPENRC}" = "true" ] && openrc_start
+    return 0
+}
+
+# --- re-evaluate args to include env command ---
+eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@")
+
+# --- run the install process --
+{
+    verify_system
+    setup_env "$@"
+    download_and_verify
+    create_symlinks
+    create_killall
+    create_uninstall
+    systemd_disable
+    create_env_file
+    create_service_file
+    service_enable_and_start
+}

k3s/master-install.sh

@@ -0,0 +1,15 @@
+sudo cp ./k3s /usr/local/bin/
+
+sudo apt-get update;
+sudo apt-get install nfs-common
+
+sudo mkdir -p /var/lib/rancher/k3s/agent/images/
+sudo cp ./k3s-airgap-images-amd64.tar /var/lib/rancher/k3s/agent/images/
+
+sudo INSTALL_K3S_SKIP_DOWNLOAD=true ./install.sh
+
+sudo rm -f ./node-token
+sudo cp /var/lib/rancher/k3s/server/node-token ./
+sudo chmod 777 ./node-token
+echo "agent token is: "
+cat ./node-token

k3s/nfs-install.sh

@@ -0,0 +1,12 @@
+apt-get udpate
+apt-get remove nfs-kernel-server
+apt-get install nfs-kernel-server
+
+read -p "ip range(such as 192.168.31.0/24): " input;
+iprange=`echo $input | tr '[A-Z]' '[a-z]'`;
+
+mkdir /data/nfs-data
+chown nobody:nogroup /data/nfs-data
+echo "/data/nfs-data $iprange(rw,sync,no_subtree_check,no_root_squash)" >> /etc/exports
+sudo systemctl restart nfs-kernel-server
+

k3s/node-token

@@ -0,0 +1 @@
+K10d6b6e8c5bc9d117e979056f03d6d533deaaf6b2b3af35f3c1a331ce91661f467::node:3152c240f50eaec6b233c550aaf0d8ef

k3s/uninstall.sh

@@ -0,0 +1 @@
+sudo /usr/local/bin/k3s-uninstall.sh

metrics-server/.gitignore

@@ -0,0 +1,12 @@
+/_output
+.cover
+
+# Vim-related files
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*~
+*.un~
+Session.vim
+.netrwhist
+.idea
+*manifest-tool

metrics-server/.golangci.yml

@@ -0,0 +1,17 @@
+run:
+  deadline: 2m
+
+linters:
+  disable-all: true
+  enable:
+    - gofmt
+    - goimports
+    - gosimple
+    - gocyclo
+    - ineffassign
+    - misspell
+    - govet
+
+linters-settings:
+  goimports:
+local-prefixes: sigs.k8s.io/metrics-server

metrics-server/.travis.yml

@@ -0,0 +1,18 @@
+sudo: false
+
+language: go
+
+services:
+  - docker
+
+go:
+- "1.12"
+
+script:
+  - make lint
+  - make test-unit
+  - make test-e2e-all
+
+before_script:
+  - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.16.2/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+  - make

metrics-server/README.md

@@ -0,0 +1,13 @@
+# Kubernetes Metrics Server
+## deploy the latest metric-server
+$ kubectl create -f ./
+clusterrole.rbac.authorization.k8s.io/system:aggregated-metrics-reader created
+clusterrolebinding.rbac.authorization.k8s.io/metrics-server:system:auth-delegator created
+rolebinding.rbac.authorization.k8s.io/metrics-server-auth-reader created
+apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io created
+serviceaccount/metrics-server created
+deployment.extensions/metrics-server created
+service/metrics-server created
+clusterrole.rbac.authorization.k8s.io/system:metrics-server created
+clusterrolebinding.rbac.authorization.k8s.io/system:metrics-server created
+

metrics-server/aggregated-metrics-reader.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:aggregated-metrics-reader
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+rules:
+- apiGroups: ["metrics.k8s.io"]
+  resources: ["pods", "nodes"]
+  verbs: ["get", "list", "watch"]

metrics-server/auth-delegator.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

metrics-server/auth-reader.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: metrics-server-auth-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

metrics-server/metrics-apiservice.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: metrics-server
+    namespace: kube-system
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: true
+  groupPriorityMinimum: 100
+  versionPriority: 100

metrics-server/metrics-server-deployment.yaml

@@ -0,0 +1,49 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    k8s-app: metrics-server
+spec:
+  selector:
+    matchLabels:
+      k8s-app: metrics-server
+  template:
+    metadata:
+      name: metrics-server
+      labels:
+        k8s-app: metrics-server
+    spec:
+      serviceAccountName: metrics-server
+      volumes:
+      # mount in tmp so we can safely use from-scratch images and/or read-only containers
+      - name: tmp-dir
+        emptyDir: {}
+      containers:
+      - name: metrics-server
+        image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server-amd64:v0.3.6
+        args:
+          - --cert-dir=/tmp
+          - --secure-port=4443
+        ports:
+        - name: main-port
+          containerPort: 4443
+          protocol: TCP
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+        imagePullPolicy: Always
+        volumeMounts:
+        - name: tmp-dir
+          mountPath: /tmp
+      nodeSelector:
+        beta.kubernetes.io/os: linux

metrics-server/metrics-server-service.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-server
+  namespace: kube-system
+  labels:
+    kubernetes.io/name: "Metrics-server"
+    kubernetes.io/cluster-service: "true"
+spec:
+  selector:
+    k8s-app: metrics-server
+  ports:
+  - port: 443
+    protocol: TCP
+    targetPort: main-port

metrics-server/resource-reader.yaml

@@ -0,0 +1,31 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  - namespaces
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: kube-system

nfs-client/README.md

@@ -0,0 +1,4 @@
+# Kubernetes Nfs Server
+## deploy the latest nfs-server
+$ kubectl create -f ./
+

nfs-client/deployment.yaml

@@ -0,0 +1,33 @@
+kind: Deployment
+apiVersion: extensions/v1beta1
+metadata:
+  name: nfs-client-provisioner
+  namespace: i2
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: nfs-client-provisioner
+    spec:
+      serviceAccountName: nfs-client-provisioner
+      containers:
+        - name: nfs-client-provisioner
+          image: quay.io/external_storage/nfs-client-provisioner:latest
+          volumeMounts:
+            - name: nfs-client-root
+              mountPath: /persistentvolumes
+          env:
+            - name: PROVISIONER_NAME
+              value: i2/nfs
+            - name: NFS_SERVER
+              value: 192.168.14.210
+            - name: NFS_PATH
+              value: /data/nfs-data
+      volumes:
+        - name: nfs-client-root
+          nfs:
+            server: 192.168.14.210
+            path: /data/nfs-data

nfs-client/rbac.yaml

@@ -0,0 +1,63 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: nfs-client-provisioner
+  namespace: i2
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nfs-client-provisioner-runner
+  namespace: i2
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: run-nfs-client-provisioner
+  namespace: i2
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    namespace: i2
+roleRef:
+  kind: ClusterRole
+  name: nfs-client-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  namespace: i2
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  namespace: i2
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    # replace with namespace where provisioner is deployed
+    namespace: i2
+roleRef:
+  kind: Role
+  name: leader-locking-nfs-client-provisioner
+  apiGroup: rbac.authorization.k8s.io

nfs-client/storageclass.yaml

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: managed-nfs-storage
+  namespace: i2
+provisioner: i2/nfs # or choose another name, must match deployment's env PROVISIONER_NAME'
+parameters:
+  archiveOnDelete: "false"

tempate/backend/service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: i2-pusher-backend
+  namespace: i2
+spec:
+  type: NodePort
+  selector:
+    app: i2-pusher-backend
+  ports:
+  - port: 8080
+    name: api
+    nodePort: 31100
+    protocol: TCP
+    targetPort: 8080
+  - port: 389
+    name: ldap
+    nodePort: 31102
+    protocol: TCP
+    targetPort: 389

tempate/backend/setup.sh

@@ -0,0 +1,2 @@
+kubectl delete statefulset i2-pusher-backend -n i2;
+kubectl apply -f ./

tempate/backend/statefulset.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: i2-pusher-backend
+  namespace: i2
+spec:
+  replicas: 1
+  volumeClaimTemplates:
+  - metadata:
+      name: i2-upload
+    spec:
+      storageClassName: managed-nfs-storage
+      accessModes: [ "ReadWriteMany" ]
+      resources:
+        requests:
+          storage: 1Gi
+  selector:
+    matchLabels:
+      app: i2-pusher-backend
+  template:
+    metadata:
+      labels:
+        app: i2-pusher-backend
+    spec:
+      hostAliases:
+        - ip: "192.168.0.1"
+          hostnames:
+          - "www.ccbeetech.com"
+          - "ccbeetech.com"
+      containers:
+      - name: i2-pusher-backend
+        image: docker.i2erp.cn/i2-pusher-backend:10951
+        volumeMounts:
+          - name: i2-upload
+            mountPath: /app/files
+            readOnly: false
+            subPath: files
+          - name: config
+            mountPath: /app/conf/
+          - name: date-config
+            mountPath: /etc/localtime
+        resources:
+          limits:
+            memory: "2Gi"
+            cpu: "1000m"
+          requests:
+            memory: "10Mi"
+            cpu: "10m"
+        ports:
+        - containerPort: 8080
+        - containerPort: 389
+      imagePullSecrets:
+        - name: registrykey-i2erp
+      volumes:
+        - name: config
+          configMap:
+            name: i2-pusher-backend
+        - name: date-config
+          hostPath:
+            path: /etc/localtime

tempate/frontend/config.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: i2-pusher-frontend
+  name: i2-pusher-frontend
+  namespace: i2
+data:
+  config.js: |-
+    window.CONFIG = {
+      AuthMode: 'local',
+      URL: 'https://api.pusher.i2erp.cn'
+    };
+  app.conf: |-
+    app_name = i2-pusher-frontend
+    http_port = 80
+    run_mode = debug
+    log_mode = debug
+    data_source = "i2pusher:i2pusher@#2019@tcp(pc-uf66645lwf7331605.mysql.polardb.rds.aliyuncs.com:3306)/i2-pusher-backend?charset=utf8&loc=Local&parseTime=true"
+    storage = http://file.qianqiusoft.com/v1/fs_file/
+    sync_db = false
+    auto_refresh = false
+    enable_ldap_server = false
+    ldap_port = 389

tempate/frontend/deployment.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: i2-pusher-frontend
+  namespace: i2
+spec:
+  selector:
+    matchLabels:
+      app: i2-pusher-frontend
+  template:
+    metadata:
+      labels:
+        app: i2-pusher-frontend
+    spec:
+      hostAliases:
+        - ip: "192.168.0.1"
+          hostnames:
+          - "www.ccbeetech.com"
+          - "ccbeetech.com"
+      containers:
+      - name: i2-pusher-frontend
+        image: docker.i2erp.cn/i2-pusher-frontend:10078
+        volumeMounts:
+          - name: config
+            mountPath: /app/web/config.js
+            subPath: config.js
+          - name: config
+            mountPath: /app/conf/app.conf
+            subPath: app.conf
+        resources:
+          limits:
+            memory: "2Gi"
+            cpu: "1000m"
+          requests:
+            memory: "10Mi"
+            cpu: "11m"
+        ports:
+        - containerPort: 80
+      imagePullSecrets:
+        - name: registrykey-i2erp
+      volumes:
+        - name: config
+          configMap:
+            name: i2-pusher-frontend

tempate/frontend/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: i2-pusher-frontend
+  namespace: i2
+spec:
+  type: NodePort
+  selector:
+    app: i2-pusher-frontend
+  ports:
+  - port: 8081
+    nodePort: 31101
+    protocol: TCP
+    targetPort: 80