diff --git a/dfdaemon/config/config.go b/dfdaemon/config/config.go index 56de62d3f..478bbc122 100644 --- a/dfdaemon/config/config.go +++ b/dfdaemon/config/config.go @@ -50,7 +50,7 @@ var fs = afero.NewOsFs() // // proxies: // # proxy all http image layer download requests with dfget -// - regx: blobs/sha256:.* +// - regx: blobs/sha256.* // # change http requests to some-registry to https and proxy them with dfget // - regx: some-registry/ // use_https: true diff --git a/dfdaemon/config/config_test.go b/dfdaemon/config/config_test.go index b384d90ff..e91d81964 100644 --- a/dfdaemon/config/config_test.go +++ b/dfdaemon/config/config_test.go @@ -252,7 +252,7 @@ func (ts *configTestSuite) TestProxyNew() { func (ts *configTestSuite) TestProxyMatch() { r := ts.Require() - p, err := NewProxy("blobs/sha256:.*", false, false) + p, err := NewProxy("blobs/sha256.*", false, false) r.Nil(err) r.NotNil(p) diff --git a/docs/quick_start/README.md b/docs/quick_start/README.md index 06f3e827b..92233274a 100644 --- a/docs/quick_start/README.md +++ b/docs/quick_start/README.md @@ -11,7 +11,7 @@ All steps in this document are done on the same machine using the docker contain ## Step 1: Deploy Dragonfly Server (SuperNode) ```bash -docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 dragonflyoss/supernode:0.4.1 +docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 -v /home/admin/supernode:/home/admin/supernode dragonflyoss/supernode:0.4.2 ``` **NOTE**: @@ -22,12 +22,12 @@ docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 dragon ```bash SUPERNODE_IP=`docker inspect supernode -f '{{.NetworkSettings.Networks.bridge.IPAddress}}'` -docker run -d --name dfclient -p 65001:65001 dragonflyoss/dfclient:0.4.1 --registry https://index.docker.io --node $SUPERNODE_IP +docker run -d --name dfclient --restart=always -p 65001:65001 -v $HOME/.small-dragonfly:/root/.small-dragonfly dragonflyoss/dfclient:0.4.2 --registry https://index.docker.io --node $SUPERNODE_IP ``` **NOTE**: -- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. +- The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the other **non-https image registries**. - The `--node` parameter specifies the supernode's ip address. Here we use `docker inspect` to get the ip of supernode container. Since the supernode container exposes its ports, you can specify this parameter to node ip address as well. ## Step 3. Configure Docker Daemon @@ -75,3 +75,12 @@ If the output of command above has content like ``` then Dragonfly is proved to work successfully. + +## SEE ALSO + +- [multi machines deployment](../user_guide/multi_machines_deployment.md) - experience Dragonfly on multiple machines +- [install server](../user_guide/install_server.md) - how to install the Dragonfly server +- [install client](../user_guide/install_client.md) - how to install the Dragonfly dfclient +- [docker proxy](../user_guide/docker_proxy.md) - make Dragonfly as HTTP proxy for docker daemon +- [proxy](../user_guide/proxy.md) - config proxy +- [download files](../user_guide/download_files.md) - download files with Dragonfly diff --git a/docs/user_guide/docker_proxy.md b/docs/user_guide/docker_proxy.md index 0424c7e22..90fe0c60d 100644 --- a/docs/user_guide/docker_proxy.md +++ b/docs/user_guide/docker_proxy.md @@ -8,7 +8,7 @@ To use dfdaemon as HTTP proxy, first you need to add a proxy rule in ```yaml proxies: -- regx: blobs/sha256:.* +- regx: blobs/sha256.* ``` This will proxy all requests for image layers with dfget. diff --git a/docs/user_guide/install_client.md b/docs/user_guide/install_client.md index c1626c55a..0a6e2fd28 100644 --- a/docs/user_guide/install_client.md +++ b/docs/user_guide/install_client.md @@ -16,23 +16,15 @@ You can install from the latest packages we provided. Available packages: - - If you're in China: + - [Linux 64-bit](https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.2/df-client_0.4.2_linux_amd64.tar.gz): `https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.2/df-client_0.4.2_linux_amd64.tar.gz` - - [Linux 64-bit](http://dragonflyoss.oss-cn-hangzhou.aliyuncs.com/df-client_0.4.0_linux_amd64.tar.gz): `http://dragonflyoss.oss-cn-hangzhou.aliyuncs.com/df-client_0.4.0_linux_amd64.tar.gz` - - - [MacOS 64-bit](http://dragonflyoss.oss-cn-hangzhou.aliyuncs.com/df-client_0.4.0_darwin_amd64.tar.gz): `http://dragonflyoss.oss-cn-hangzhou.aliyuncs.com/df-client_0.4.0_darwin_amd64.tar.gz` - - - If you're not in China: - - - [Linux 64-bit](https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.0/df-client_0.4.0_linux_amd64.tar.gz): `https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.0/df-client_0.4.0_linux_amd64.tar.gz` - - - [MacOS 64-bit](https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.0/df-client_0.4.0_darwin_amd64.tar.gz): `https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.0/df-client_0.4.0_darwin_amd64.tar.gz` + - [MacOS 64-bit](https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.2/df-client_0.4.2_darwin_amd64.tar.gz): `https://github.com/dragonflyoss/Dragonfly/releases/download/v0.4.2/df-client_0.4.2_darwin_amd64.tar.gz` 2. Unzip the package. ```bash # Replace `xxx` with the installation directory. - tar -zxf df-client_0.4.0_linux_amd64.tar.gz -C xxx + tar -zxf df-client_0.4.2_linux_amd64.tar.gz -C xxx ``` 3. Add the directory of `df-client` to your `PATH` environment variable to make sure you can directly use `dfget` and `dfdaemon` command. @@ -72,13 +64,13 @@ You can also install from the source code. 4. Install `dfdaemon` and `dfget` in `/opt/dragonfly/df-client` and create soft-link in `/usr/local/bin`. ```sh - sudo make install + sudo make install-client ``` ## After this Task Test if the downloading works. - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" - ``` +```sh +dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" +``` diff --git a/docs/user_guide/install_server.md b/docs/user_guide/install_server.md index e6732df71..c7debd55a 100644 --- a/docs/user_guide/install_server.md +++ b/docs/user_guide/install_server.md @@ -1,12 +1,12 @@ -# Installing Server +# Installing Dragonfly Server -This topic explains how to install the Dragonfly server. +This topic explains how to install the Dragonfly server with **Golang version**. -**Tip:** For a data center or a cluster, we recommend that you use at least two machines with eight cores, 16GB RAM and Gigabit Ethernet connections for deploying supernodes. +**NOTE**: The Golang version supernode is **not ready for production usage**. However, you can use it more easily in your test environment. ## Context -There are two layers in Dragonfly’s architecture: server (supernodes) and client (hosts). Install the supernodes in one of the following ways: +Install the SuperNodes in one of the following ways: - Deploying with Docker: Recommended for quick local deployment and test. - Deploying with physical machines: Recommended for production usage. @@ -25,12 +25,23 @@ When deploying with physical machines, the following conditions must be met. Required Software | Version Limit ---|--- Git|1.9.1+ -JDK|1.7+ -Maven|3.0.3+ +Golang|1.12.x Nginx|0.8+ ## Procedure - When Deploying with Docker +### Get SuperNode image + +You can get it from [DockerHub](https://hub.docker.com/) directly. + +1. Obtain the latest Docker image ID of the SuperNode. + + ```sh + docker pull dragonflyoss/supernode:0.4.2 + ``` + +Or you can build your own supernode image. + 1. Obtain the source code of Dragonfly. ```sh @@ -46,21 +57,27 @@ Nginx|0.8+ 3. Build the Docker image. ```sh - make build-supernode-java + TAG="0.4.2" + make docker-build-supernode DF_VERSION=$TAG ``` -4. Obtain the latest Docker image ID of the supernode. +4. Obtain the latest Docker image ID of the SuperNode. ```sh docker image ls|grep 'supernode' |awk '{print $3}' | head -n1 ``` -5. Start the supernode. +### Start the SuperNode - ```sh - # Replace ${supernodeDockerImageId} with the ID obtained at the previous step - docker run -d -p 8001:8001 -p 8002:8002 ${supernodeDockerImageId} - ``` +**NOTE**: Replace ${supernodeDockerImageId} with the ID obtained at the previous step. + +```sh +docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 -v /home/admin/supernode:/home/admin/supernode dragonflyoss/supernode:0.4.2 --download-port=8001 + +or + +docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 -v /home/admin/supernode:/home/admin/supernode ${supernodeDockerImageId} --download-port=8001 +``` ## Procedure - When Deploying with Physical Machines @@ -73,25 +90,24 @@ Nginx|0.8+ 2. Enter the project directory. ```sh - cd Dragonfly/src/supernode + cd Dragonfly ``` 3. Compile the source code. ```sh - mvn clean -U install -DskipTests=true + make build-supernode && make install ``` -4. Start the supernode. +4. Start the SuperNode. ```sh - # If the 'supernode.baseHome’ is not specified, then the default value '/home/admin/supernode’ will be used. - java -Dsupernode.baseHome=/home/admin/supernode -jar target/supernode.jar + supernode --home-dir=/home/admin/supernode --port=8002 --download-port=8001 --advertise-ip=127.0.0.1 ``` -5. Add the following configuration items to the Nginx configuration file. + **NOTE**: `advertise-ip` should be the ip that clients can connect to, `127.0.0.1` here is an example for testing, and it can only be used if the server and client are in the same machine. - **Tip:** The path of the Nginx configuration file is something like `src/supernode/src/main/docker/sources/nginx.conf`. +5. Add the following configuration items to the Nginx configuration file. ```conf server { @@ -101,13 +117,6 @@ Nginx|0.8+ root /home/admin/supernode/repo; } } - - server { - listen 8002; - location / { - proxy_pass http://127.0.0.1:8080; - } - } ``` 6. Start Nginx. @@ -118,17 +127,15 @@ Nginx|0.8+ ## After this Task -- After the supernode is installed, run the following commands to verify if Nginx and Tomcat are started, and if Port `8001` and `8002` are available. +- After the SuperNode is installed, run the following commands to verify if Nginx and **Supernode** are started, and if Port `8001` and `8002` are available. ```sh - ps aux|grep nginx - ps aux|grep tomcat telnet 127.0.0.1 8001 - telent 127.0.0.1 8002 + telnet 127.0.0.1 8002 ``` - Install the Dragonfly client and test if the downloading works. ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1" + dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" ``` diff --git a/docs/user_guide/install_server_go.md b/docs/user_guide/install_server_go.md deleted file mode 100644 index 944dcee38..000000000 --- a/docs/user_guide/install_server_go.md +++ /dev/null @@ -1,135 +0,0 @@ -# Installing Dragonfly Server - -This topic explains how to install the Dragonfly server with **Golang version**. - -**NOTE**: The Golang version supernode is **not ready for production usage**. However, you can use it more easily in your test environment. - -## Context - -Install the SuperNodes in one of the following ways: - -- Deploying with Docker: Recommended for quick local deployment and test. -- Deploying with physical machines: Recommended for production usage. - -## Prerequisites - -When deploying with Docker, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Docker|1.12.0+ - -When deploying with physical machines, the following conditions must be met. - -Required Software | Version Limit ----|--- -Git|1.9.1+ -Golang|1.12.x -Nginx|0.8+ - -## Procedure - When Deploying with Docker - -### Get Supernode image - -You can get it from [DockerHub](https://hub.docker.com/) directly. - -1. Obtain the latest Docker image ID of the SuperNode. - - ```sh - docker pull dragonflyoss/supernode:0.4.0 - ``` - -Or you can build your own supernode image. - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Build the Docker image. - - ```sh - TAG="test" - make docker-build-supernode DF_VERSION=$TAG - ``` - -4. Obtain the latest Docker image ID of the SuperNode. - - ```sh - docker image ls|grep 'supernode' |awk '{print $3}' | head -n1 - ``` - -### Start the SuperNode - -**NOTE**: Replace ${supernodeDockerImageId} with the ID obtained at the previous step. - -```sh -docker run -d --name dfsupernode -p 8002:8002 -v /home/admin/supernode:/home/admin/supernode ${supernodeDockerImageId} --advertise-ip=127.0.0.1 --download-port=8001 -``` - -## Procedure - When Deploying with Physical Machines - -1. Obtain the source code of Dragonfly. - - ```sh - git clone https://github.com/dragonflyoss/Dragonfly.git - ``` - -2. Enter the project directory. - - ```sh - cd Dragonfly - ``` - -3. Compile the source code. - - ```sh - make build-supernode && make install - ``` - -4. Start the SuperNode. - - ```sh - supernode --home-dir=/home/admin/supernode --port=8002 --advertise-ip=127.0.0.1 --download-port=8001 - ``` - -5. Add the following configuration items to the Nginx configuration file. - - ```conf - server { - listen 8001; - location / { - # Must be ${supernode.baseHome}/repo - root /home/admin/supernode/repo; - } - } - ``` - -6. Start Nginx. - - ```sh - sudo nginx - ``` - -## After this Task - -- After the SuperNode is installed, run the following commands to verify if Nginx and **Supernode** are started, and if Port `8001` and `8002` are available. - - ```sh - telnet 127.0.0.1 8001 - telnet 127.0.0.1 8002 - ``` - -- Install the Dragonfly client and test if the downloading works. - - ```sh - dfget --url "http://${resourceUrl}" --output ./resource.png --node "127.0.0.1:8002" - ``` diff --git a/docs/user_guide/multi_machines_deployment.md b/docs/user_guide/multi_machines_deployment.md index 93feaae28..9f027de8d 100644 --- a/docs/user_guide/multi_machines_deployment.md +++ b/docs/user_guide/multi_machines_deployment.md @@ -25,11 +25,10 @@ Then, we must provice: Deploy the Dragonfly server (Supernode) on the machine `dfsupernode`. ```bash -docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 \ - dragonflyoss/supernode:0.3.0 -Dsupernode.advertiseIp=dfsupernode +docker run -d --name supernode --restart=always -p 8001:8001 -p 8002:8002 -v /home/admin/supernode:/home/admin/supernode dragonflyoss/supernode:0.4.2 --download-port=8001 --advertise-ip=dfsupernode ``` -> **NOTE**: `supernode.advertiseIp` should be the ip that clients can connect to, `127.0.0.1` here is an example for testing, and it can only be used if the server and client are in the same machine. +> **NOTE**: `advertise-ip` should be the ip that clients can connect to. ## Step 2：Deploy Dragonfly Client (dfclient) @@ -53,7 +52,8 @@ EOD ```bash docker run -d --name dfclient --restart=always -p 65001:65001 \ -v /etc/dragonfly:/etc/dragonfly \ - dragonflyoss/dfclient:0.4.0 --registry https://index.docker.io + -v $HOME/.small-dragonfly:/root/.small-dragonfly \ + dragonflyoss/dfclient:0.4.2 --registry https://index.docker.io ``` **NOTE**: The `--registry` parameter specifies the mirrored image registry address, and `https://index.docker.io` is the address of official image registry, you can also set it to the others. diff --git a/docs/user_guide/proxy.md b/docs/user_guide/proxy.md index d1a9df3cd..04dc31ccc 100644 --- a/docs/user_guide/proxy.md +++ b/docs/user_guide/proxy.md @@ -14,7 +14,7 @@ Proxy rules are configured in `/etc/dragonfly/dfdaemon.yml`. # matching rule. proxies: # proxy all http image layer download requests with dfget -- regx: blobs/sha256:.* +- regx: blobs/sha256.* # proxy requests directly, without dfget - regx: no-proxy-reg direct: true diff --git a/docs/user_guide/supernode_configuration.md b/docs/user_guide/supernode_configuration.md deleted file mode 100644 index 6ab4249bd..000000000 --- a/docs/user_guide/supernode_configuration.md +++ /dev/null @@ -1,62 +0,0 @@ -# Supernode Configuration - -The supernode is written in Java based on Spring Boot. You can easily set properties with command line parameters or with the configuration file. - - -## Supernode Properties - -### Simple Property - -Property Name | Default Value | Description ----|---|--- -supernode.baseHome | /home/admin/supernode | Working directory of the supernode -supernode.systemNeedRate | 20 | Network rate reserved for the system (Unit: MB/s) -supernode.totalLimit | 200 | Network rate reserved for the supernode (Unit: MB/s) -supernode.schedulerCorePoolSize | 10 | Core pool size of ScheduledExecutorService -supernode.dfgetPath | /usr/local/bin/dfget/ | The `dfget` path - -### Cluster Property - -#### supernode.cluster - -This is an array property, and every member of it has these attributes: - -Name | Default Value | Description ----- | ------------- | ----------- -ip | None | The ip of the cluster member. -downloadPort | 8001 | The download port of the cluster member. -registerPort | 8002 | The register port of the cluster member. - -- Config it in `.properties` file, for example: - - ```ini - supernode.cluster[0].ip = '192.168.0.1' - supernode.cluster[0].registerPort = 8002 - supernode.cluster[1].ip = '192.168.0.2' - ``` - -- Config it in `.yaml` file, for example: - - ```yaml - supernode: - cluster: - - ip: '192.168.0.1' - registerPort: 8002 - - ip: '192.168.0.2' - ``` - -## Setting Properties - -You have two options when setting properties of a supernode. - -- Setting properties with command line parameters. - - ```bash - java -D= -jar supernode.jar - ``` - -- Setting properties with the configuration file. - - ```bash - java -Dspring.config.location=./config.properties, -jar supernode.jar - ```