updated Dockerfile

.dockerignore

@@ -1,4 +1,5 @@
 .idea
 logs
 *.log
-node_modules/
+dist/
+**/node_modules/

Dockerfile

@@ -1,45 +1,59 @@
+FROM golang:1.12 AS backend-build
+
+WORKDIR /go/src/app
+COPY ./backend .
+
+ENV GO111MODULE on
+ENV GOPROXY https://mirrors.aliyun.com/goproxy/
+
+RUN go install -v ./...
+
+FROM node:8.16.0-alpine AS frontend-build
+
+ADD ./frontend /app
+WORKDIR /app
+
+# install frontend
+RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org
+
+RUN npm run build:prod
+
 # images
 FROM ubuntu:latest
 
-# source files
-ADD . /opt/crawlab
+ADD . /app
 
 # set as non-interactive
 ENV DEBIAN_FRONTEND noninteractive
 
-# environment variables
-ENV NVM_DIR /usr/local/nvm  
-ENV NODE_VERSION 8.12.0
-ENV WORK_DIR /opt/crawlab
-
-# install pkg
+# install packages
 RUN apt-get update \
-	&& apt-get install -y curl git net-tools iputils-ping ntp nginx python3 python3-pip \
-	&& apt-get clean \
-	&& cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \
 	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
 	&& ln -s /usr/bin/python3 /usr/local/bin/python
 
-# install nvm
-RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \  
-    && . $NVM_DIR/nvm.sh \
-    && nvm install v$NODE_VERSION \
-    && nvm use v$NODE_VERSION \
-    && nvm alias default v$NODE_VERSION
-ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules  
-ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+# install backend
+RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple
 
-# install frontend
-RUN npm install -g yarn \
-	&& cd /opt/crawlab/frontend \
-	&& yarn install
+# copy backend files
+COPY --from=backend-build /go/src/app .
+COPY --from=backend-build /go/bin/crawlab /usr/local/bin
 
-# install backend
-RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \
-	&& pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+# install nginx
+RUN apt-get -y install nginx
 
-# start backend
+# copy frontend files
+COPY --from=frontend-build /app/dist /app/dist
+COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
+
+# working directory
+WORKDIR /app/backend
+
+# frontend port
 EXPOSE 8080
+
+# backend port
 EXPOSE 8000
-WORKDIR /opt/crawlab
-ENTRYPOINT ["/bin/sh", "/opt/crawlab/docker_init.sh"]
+
+# start backend
+CMD ["/bin/sh", "/app/docker_init.sh"]

backend/conf/config.yml

@@ -1,7 +1,11 @@
+api:
+  address: "localhost:8000"
 mongo:
   host: localhost
   port: 27017
   db: crawlab_test
+  username: ""
+  password: ""
 redis:
   network: tcp
   address: "localhost:6379"
@@ -11,10 +15,10 @@ log:
 server:
   host: 0.0.0.0
   port: 8000
-  master: "Y"
+  master: "N"
   secret: "crawlab"
 spider:
-  path: "/Users/yeqing/projects/crawlab/spiders"
+  path: "/app/spiders"
 task:
   workers: 4
 other:

backend/model/node.go

@@ -21,8 +21,9 @@ type Node struct {
 	// 前端展示
 	IsMaster bool `json:"is_master"`
 
-	UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
-	CreateTs time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTs     time.Time `json:"update_ts" bson:"update_ts"`
+	CreateTs     time.Time `json:"create_ts" bson:"create_ts"`
+	UpdateTsUnix int64       `json:"update_ts_unix" bson:"update_ts_unix"`
 }
 
 func (n *Node) Save() error {
@@ -40,6 +41,7 @@ func (n *Node) Add() error {
 	defer s.Close()
 	n.Id = bson.NewObjectId()
 	n.UpdateTs = time.Now()
+	n.UpdateTsUnix = time.Now().Unix()
 	n.CreateTs = time.Now()
 	if err := c.Insert(&n); err != nil {
 		debug.PrintStack()

backend/services/node.go

@@ -16,10 +16,11 @@ import (
 )
 
 type Data struct {
-	Mac      string    `json:"mac"`
-	Ip       string    `json:"ip"`
-	Master   bool      `json:"master"`
-	UpdateTs time.Time `json:"update_ts"`
+	Mac          string    `json:"mac"`
+	Ip           string    `json:"ip"`
+	Master       bool      `json:"master"`
+	UpdateTs     time.Time `json:"update_ts"`
+	UpdateTsUnix int64     `json:"update_ts_unix"`
 }
 
 type NodeMessage struct {
@@ -193,9 +194,8 @@ func UpdateNodeStatus() {
 		}
 
 		// 如果记录的更新时间超过60秒，该节点被认为离线
-		if time.Now().Sub(data.UpdateTs) > 60*time.Second {
+		if time.Now().Unix()-data.UpdateTsUnix > 60 {
 			// 在Redis中删除该节点
-
 			if err := database.RedisClient.HDel("nodes", data.Mac); err != nil {
 				log.Errorf(err.Error())
 				return
@@ -284,10 +284,11 @@ func UpdateNodeData() {
 
 	// 构造节点数据
 	data := Data{
-		Mac:      mac,
-		Ip:       ip,
-		Master:   IsMaster(),
-		UpdateTs: time.Now(),
+		Mac:          mac,
+		Ip:           ip,
+		Master:       IsMaster(),
+		UpdateTs:     time.Now(),
+		UpdateTsUnix: time.Now().Unix(),
 	}
 
 	// 注册节点到Redis

backend/services/task.go

@@ -124,10 +124,15 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e
 	cmd.Stdout = fLog
 	cmd.Stderr = fLog
 
-	// 添加环境变量
+	// 添加默认环境变量
 	cmd.Env = append(cmd.Env, "CRAWLAB_TASK_ID="+t.Id)
 	cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+s.Col)
 
+	// 添加任务环境变量
+	for _, env := range s.Envs {
+		cmd.Env = append(cmd.Env, env.Name + "=" + env.Value)
+	}
+
 	// 起一个goroutine来监控进程
 	ch := TaskExecChanMap.ChanBlocked(t.Id)
 	go func() {
@@ -393,7 +398,7 @@ func GetTaskLog(id string) (logStr string, err error) {
 	}
 
 	logStr = ""
-	if IsMaster() {
+	if IsMasterNode(task.NodeId.Hex()) {
 		// 若为主节点，获取本机日志
 		logBytes, err := GetLocalLog(task.LogPath)
 		logStr = string(logBytes)

docker_init.sh

@@ -1,16 +1,10 @@
 #!/bin/sh
-case $1 in
-	master)
-		cd $WORK_DIR/frontend \
-			&& npm run build:prod \
-			&& service nginx start
-		python $WORK_DIR/crawlab/flower.py >> /opt/crawlab/flower.log 2>&1 &
-		python $WORK_DIR/crawlab/worker.py >> /opt/crawlab/worker.log 2>&1 &
-		cd $WORK_DIR/crawlab \
-			&& gunicorn --log-level=DEBUG -b 0.0.0.0 -w 8 app:app
-			;;
-	worker)
-		python $WORK_DIR/crawlab/app.py >> /opt/crawlab/app.log 2>&1 &
-		python $WORK_DIR/crawlab/worker.py	
-		;;
-esac
+
+# replace default api path to new one
+jspath=`ls /app/dist/js/app.*.js`
+cat ${jspath} | sed "s/localhost:8000/${CRAWLAB_API_ADDRESS}/g" > ${jspath}
+
+# start nginx
+service nginx start
+
+crawlab

frontendsrc/views/layout/components/Sidebar/index.vue

@@ -0,0 +1,75 @@
+<template>
+  <el-scrollbar wrap-class="scrollbar-wrapper">
+    <div class="sidebar-logo" :class="isCollapse ? 'collapsed' : ''">
+      <span>C</span><span v-show="!isCollapse">rawlab</span>
+    </div>
+    <el-menu
+      :show-timeout="200"
+      :default-active="routeLevel1"
+      :collapse="isCollapse"
+      :background-color="variables.menuBg"
+      :text-color="variables.menuText"
+      :active-text-color="variables.menuActiveText"
+      mode="vertical"
+    >
+      <sidebar-item v-for="route in routes" :key="route.path" :item="route" :base-path="route.path"/>
+    </el-menu>
+  </el-scrollbar>
+</template>
+
+<script>
+import { mapState, mapGetters } from 'vuex'
+import variables from '@/styles/variables.scss'
+import SidebarItem from './SidebarItem'
+
+export default {
+  components: { SidebarItem },
+  computed: {
+    ...mapState('user', [
+      'adminPaths'
+    ]),
+    ...mapGetters([
+      'sidebar'
+    ]),
+    routeLevel1 () {
+      let pathArray = this.$route.path.split('/')
+      return `/${pathArray[1]}`
+    },
+    routes () {
+      return this.$router.options.routes.filter(d => {
+        const role = this.$store.getters['user/userInfo'].role
+        if (role === 'admin') return true
+        return !this.adminPaths.includes(d.path)
+      })
+    },
+    variables () {
+      return variables
+    },
+    isCollapse () {
+      return !this.sidebar.opened
+    }
+  }
+}
+</script>
+
+<style>
+  #app .sidebar-container .el-menu {
+    height: calc(100% - 50px);
+  }
+
+  .sidebar-container .sidebar-logo {
+    height: 50px;
+    display: flex;
+    align-items: center;
+    padding-left: 20px;
+    color: #fff;
+    background: rgb(48, 65, 86);
+    font-size: 28px;
+    font-weight: 600;
+    font-family: "Verdana", serif;
+  }
+
+  .sidebar-container .sidebar-logo.collapsed {
+    padding-left: 7px;
+  }
+</style>