diff --git a/.dockerignore b/.dockerignore index 5900766d7..4207a9513 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ .idea logs *.log -node_modules/ \ No newline at end of file +dist/ +**/node_modules/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3b07591da..fc10b586d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,45 +1,59 @@ +FROM golang:1.12 AS backend-build + +WORKDIR /go/src/app +COPY ./backend . + +ENV GO111MODULE on +ENV GOPROXY https://mirrors.aliyun.com/goproxy/ + +RUN go install -v ./... + +FROM node:8.16.0-alpine AS frontend-build + +ADD ./frontend /app +WORKDIR /app + +# install frontend +RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org + +RUN npm run build:prod + # images FROM ubuntu:latest -# source files -ADD . /opt/crawlab +ADD . /app # set as non-interactive ENV DEBIAN_FRONTEND noninteractive -# environment variables -ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8.12.0 -ENV WORK_DIR /opt/crawlab - -# install pkg +# install packages RUN apt-get update \ - && apt-get install -y curl git net-tools iputils-ping ntp nginx python3 python3-pip \ - && apt-get clean \ - && cp $WORK_DIR/crawlab.conf /etc/nginx/conf.d \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python -# install nvm -RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.24.0/install.sh | bash \ - && . $NVM_DIR/nvm.sh \ - && nvm install v$NODE_VERSION \ - && nvm use v$NODE_VERSION \ - && nvm alias default v$NODE_VERSION -ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules -ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# install backend +RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple -# install frontend -RUN npm install -g yarn \ - && cd /opt/crawlab/frontend \ - && yarn install +# copy backend files +COPY --from=backend-build /go/src/app . +COPY --from=backend-build /go/bin/crawlab /usr/local/bin -# install backend -RUN pip install -U setuptools -i https://pypi.tuna.tsinghua.edu.cn/simple \ - && pip install -r /opt/crawlab/crawlab/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +# install nginx +RUN apt-get -y install nginx -# start backend +# copy frontend files +COPY --from=frontend-build /app/dist /app/dist +COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# working directory +WORKDIR /app/backend + +# frontend port EXPOSE 8080 + +# backend port EXPOSE 8000 -WORKDIR /opt/crawlab -ENTRYPOINT ["/bin/sh", "/opt/crawlab/docker_init.sh"] \ No newline at end of file + +# start backend +CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 0d575bfc1..3e7aa234d 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -1,7 +1,11 @@ +api: + address: "localhost:8000" mongo: host: localhost port: 27017 db: crawlab_test + username: "" + password: "" redis: network: tcp address: "localhost:6379" @@ -11,10 +15,10 @@ log: server: host: 0.0.0.0 port: 8000 - master: "Y" + master: "N" secret: "crawlab" spider: - path: "/Users/yeqing/projects/crawlab/spiders" + path: "/app/spiders" task: workers: 4 other: diff --git a/backend/model/node.go b/backend/model/node.go index 7bdffb1c7..01bbd3f71 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -21,8 +21,9 @@ type Node struct { // 前端展示 IsMaster bool `json:"is_master"` - UpdateTs time.Time `json:"update_ts" bson:"update_ts"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTs time.Time `json:"update_ts" bson:"update_ts"` + CreateTs time.Time `json:"create_ts" bson:"create_ts"` + UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"` } func (n *Node) Save() error { @@ -40,6 +41,7 @@ func (n *Node) Add() error { defer s.Close() n.Id = bson.NewObjectId() n.UpdateTs = time.Now() + n.UpdateTsUnix = time.Now().Unix() n.CreateTs = time.Now() if err := c.Insert(&n); err != nil { debug.PrintStack() diff --git a/backend/services/node.go b/backend/services/node.go index 0c2c95276..90f941e4b 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -16,10 +16,11 @@ import ( ) type Data struct { - Mac string `json:"mac"` - Ip string `json:"ip"` - Master bool `json:"master"` - UpdateTs time.Time `json:"update_ts"` + Mac string `json:"mac"` + Ip string `json:"ip"` + Master bool `json:"master"` + UpdateTs time.Time `json:"update_ts"` + UpdateTsUnix int64 `json:"update_ts_unix"` } type NodeMessage struct { @@ -193,9 +194,8 @@ func UpdateNodeStatus() { } // 如果记录的更新时间超过60秒，该节点被认为离线 - if time.Now().Sub(data.UpdateTs) > 60*time.Second { + if time.Now().Unix()-data.UpdateTsUnix > 60 { // 在Redis中删除该节点 - if err := database.RedisClient.HDel("nodes", data.Mac); err != nil { log.Errorf(err.Error()) return @@ -284,10 +284,11 @@ func UpdateNodeData() { // 构造节点数据 data := Data{ - Mac: mac, - Ip: ip, - Master: IsMaster(), - UpdateTs: time.Now(), + Mac: mac, + Ip: ip, + Master: IsMaster(), + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), } // 注册节点到Redis diff --git a/backend/services/task.go b/backend/services/task.go index 51b596a7c..4b75a0dad 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -124,10 +124,15 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e cmd.Stdout = fLog cmd.Stderr = fLog - // 添加环境变量 + // 添加默认环境变量 cmd.Env = append(cmd.Env, "CRAWLAB_TASK_ID="+t.Id) cmd.Env = append(cmd.Env, "CRAWLAB_COLLECTION="+s.Col) + // 添加任务环境变量 + for _, env := range s.Envs { + cmd.Env = append(cmd.Env, env.Name + "=" + env.Value) + } + // 起一个goroutine来监控进程 ch := TaskExecChanMap.ChanBlocked(t.Id) go func() { @@ -393,7 +398,7 @@ func GetTaskLog(id string) (logStr string, err error) { } logStr = "" - if IsMaster() { + if IsMasterNode(task.NodeId.Hex()) { // 若为主节点，获取本机日志 logBytes, err := GetLocalLog(task.LogPath) logStr = string(logBytes) diff --git a/docker_init.sh b/docker_init.sh index 2404580cf..587e8af32 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -1,16 +1,10 @@ #!/bin/sh -case $1 in - master) - cd $WORK_DIR/frontend \ - && npm run build:prod \ - && service nginx start - python $WORK_DIR/crawlab/flower.py >> /opt/crawlab/flower.log 2>&1 & - python $WORK_DIR/crawlab/worker.py >> /opt/crawlab/worker.log 2>&1 & - cd $WORK_DIR/crawlab \ - && gunicorn --log-level=DEBUG -b 0.0.0.0 -w 8 app:app - ;; - worker) - python $WORK_DIR/crawlab/app.py >> /opt/crawlab/app.log 2>&1 & - python $WORK_DIR/crawlab/worker.py - ;; -esac \ No newline at end of file + +# replace default api path to new one +jspath=`ls /app/dist/js/app.*.js` +cat ${jspath} | sed "s/localhost:8000/${CRAWLAB_API_ADDRESS}/g" > ${jspath} + +# start nginx +service nginx start + +crawlab \ No newline at end of file diff --git "a/frontend\032src/views/layout/components/Sidebar/index.vue" "b/frontend\032src/views/layout/components/Sidebar/index.vue" new file mode 100644 index 000000000..9388680b3 --- /dev/null +++ "b/frontend\032src/views/layout/components/Sidebar/index.vue" @@ -0,0 +1,75 @@ + + + + + diff --git "a/frontend\032src/views/login/index.vue" "b/frontend\032src/views/login/index.vue" new file mode 100644 index 000000000..f21507944 --- /dev/null +++ "b/frontend\032src/views/login/index.vue" @@ -0,0 +1,411 @@ + + + + + + + diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1 @@ +node_modules/ diff --git a/frontend/.env.production b/frontend/.env.production index 58fe7f33d..ccefa580a 100644 --- a/frontend/.env.production +++ b/frontend/.env.production @@ -1,2 +1,2 @@ NODE_ENV='production' -VUE_APP_BASE_URL='http://114.67.75.98:8000/api' +VUE_APP_BASE_URL='http://localhost:8000/api' diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 000000000..52ecb5495 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,23 @@ +FROM node:8.16.0-alpine AS frontend-build + +ADD . /app +WORKDIR /app + +# install frontend +RUN npm install -g yarn \ + && yarn install --registry=https://registry.npm.taobao.org + +RUN npm run build:prod + +FROM alpine + +#RUN apk update +RUN apk add nginx +COPY --from=frontend-build /app/dist /app/dist +COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d +#RUN nginx -s start +#COPY ./dist /usr/share/nginx/html + +#EXPOSE 80 +#EXPOSE 8080 + diff --git a/frontend/conf/crawlab.conf b/frontend/conf/crawlab.conf new file mode 100644 index 000000000..c3da703f3 --- /dev/null +++ b/frontend/conf/crawlab.conf @@ -0,0 +1,5 @@ +server { + listen 8080; + root /app/dist; + index index.html; +} diff --git a/frontend/src/assets/logo.png b/frontend/src/assets/logo.png new file mode 100644 index 000000000..b7473016a Binary files /dev/null and b/frontend/src/assets/logo.png differ diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index e4339dc50..e9cef8386 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -4,7 +4,7 @@ - {{$t('User')}} + {{username}} @@ -19,12 +19,12 @@ - - English - 中文 + + English + @@ -44,7 +44,12 @@ export default { ...mapGetters([ 'sidebar', 'avatar' - ]) + ]), + username () { + if (!this.$store.getters['user/userInfo']) return this.$t('User') + if (!this.$store.getters['user/userInfo'].username) return this.$t('User') + return this.$store.getters['user/userInfo'].username + } }, methods: { toggleSideBar () {