diff --git a/.github/workflows/local_tests.yml b/.github/workflows/local_tests.yml index 6f01b98de03..1da5e317390 100644 --- a/.github/workflows/local_tests.yml +++ b/.github/workflows/local_tests.yml @@ -7,12 +7,12 @@ name: Local jobs: local_build_container: - runs-on: ubuntu:22.04 + runs-on: panda-arc steps: - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Build docker container from project root - run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} . + run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local:${{ github.sha }} . - name: Minimal test of built container # Just test to see if one of our binaries is built - run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' + run: docker run --rm "panda_local:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' diff --git a/.github/workflows/parallel_tests.yml b/.github/workflows/parallel_tests.yml index b4dc7c88644..a39bdb7af78 100644 --- a/.github/workflows/parallel_tests.yml +++ b/.github/workflows/parallel_tests.yml @@ -1,8 +1,7 @@ name: Parallel Tests # For PRs to dev or pushes that modify the root Dockerfile, build from scratch # then run CI tests using that container in parallel -# For forked repos that can't use our self-hosted test suite, just build and run make check - +# For forked repos that can't use our panda-arc test suite, just build and run make check on: pull_request: branches: @@ -17,14 +16,24 @@ on: jobs: test_installer: # test install_ubuntu.sh - runs-on: ubuntu-20.04 # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet) + runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet) steps: + - name: Update + run: sudo apt-get update -y + - name: Install ssl + run: sudo apt-get install -y libssl-dev + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Install Python dev headers + run: sudo apt-get install -y libpython3-dev - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - name: Lint PyPANDA with flake8 run: | - python -m pip install --upgrade pip - python -m pip install flake8 - python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics + pip install --upgrade pip + pip install flake8 + flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics # python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run install_ubuntu.sh run: cd $GITHUB_WORKSPACE && ./panda/scripts/install_ubuntu.sh @@ -32,105 +41,137 @@ jobs: build_container: if: github.repository == 'panda-re/panda' - runs-on: self-hosted + runs-on: panda-arc steps: - - - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory - - - name: Build docker container from project root - run: cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} . - - - name: Minimal test of built container # Just test to see if one of our binaries is built - run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' - - taint_tests: + - name: Install git + run: sudo apt-get update -y && sudo apt-get install git -y + - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: ${{ github.workspace }} + tags: ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }} + target: developer + - name: Minimal test of built container # Just test to see if one of our binaries is built + run: docker run --rm "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")' + + tests: if: github.repository == 'panda-re/panda' - runs-on: self-hosted + runs-on: panda-arc needs: [build_container] strategy: matrix: - target: [i386, x86_64] + include: + - test_type: "taint" + target: "i386" + - test_type: "taint" + target: "x86_64" + - test_type: "sym_trace" + target: "x86_64" + - test_type: "pypanda" + test_script: "dyn_hooks" + - test_type: "pypanda" + test_script: "copy_test" + - test_type: "pypanda" + test_script: "file_fake" + - test_type: "pypanda" + test_script: "file_hook" + - test_type: "pypanda" + test_script: "generic_tests" + - test_type: "pypanda" + test_script: "monitor_cmds" + - test_type: "pypanda" + test_script: "multi_proc_cbs" + - test_type: "pypanda" + test_script: "sleep_in_cb" + - test_type: "pypanda" + test_script: "syscalls" + - test_type: "pypanda" + test_script: "record_no_snap" + - test_type: "pypanda" + test_script: "sig_suppress" + - test_type: "make_check" + test_script: "check-qtest-x86_64" + - test_type: "make_check" + test_script: "check-qtest-i386" + - test_type: "make_check" + test_script: "check-qtest-arm" + - test_type: "make_check" + test_script: "check-qtest-mips" + - test_type: "make_check" + test_script: "check-qtest-mipsel" + - test_type: "make_check" + test_script: "check-qtest-ppc" + - test_type: "make_check" + test_script: "check-block" + - test_type: "make_check" + test_script: "check-unit" + - test_type: "make_check" + test_script: "check-qapi-schema" + steps: # Given a container with PANDA installed at /panda, run the taint tests - - name: Run taint tests inside current container + - name: Update + run: sudo apt-get update -y + - name: Install ssl + run: sudo apt-get install -y wget + - name: Run Taint Tests + if: matrix.test_type == 'taint' run: >- + wget -q -O wheezy_panda2.qcow2 https://panda-re.mit.edu/qcows/linux/debian/7.3/x86/debian_7.3_x86.qcow; + wget -q https://panda-re.mit.edu/qcows/linux/ubuntu/1804/x86_64/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2; docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID} - --mount type=bind,source=/home/panda/regdir/qcows/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2 - --mount type=bind,source=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2 - --rm -t "panda_local_${{ github.sha }}" bash -c + --mount type=bind,source=$(pwd)/wheezy_panda2.qcow2,target=/home/panda/regdir/qcows/wheezy_panda2.qcow2 + --mount type=bind,source=$(pwd)/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2,target=/home/panda/regdir/qcows/bionic-server-cloudimg-amd64-noaslr-nokaslr.qcow2 + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c "cd /tmp; git clone https://github.com/panda-re/panda_test; cd ./panda_test/tests/taint2; + echo 'Running Record:'; python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode record; + echo 'Running Replay:'; python3 taint2_multi_arch_record_or_replay.py --arch ${{ matrix.target }} --mode replay; sed -i '/^\s*$/d' taint2_log; if cat taint2_log; then echo 'Taint unit test log found!'; else echo 'Taint unit test log NOT found!' && exit 1; fi; echo -e '\nFailures:'; if grep 'fail' taint2_log; then echo 'TEST FAILED!' && exit 1; else echo -e 'None.\nTEST PASSED!' && exit 0; fi" - sym_trace_tests: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - target: [x86_64] - - steps: - # Given a container with PANDA installed at /panda, run the taint tests - - name: Run symbolic tracing tests inside current container + - name: Run symbolic tracing tests + if: matrix.test_type == 'sym_trace' run: >- docker run --name panda_sym_test_${{ matrix.target }}_${GITHUB_RUN_ID} - --rm -t "panda_local_${{ github.sha }}" bash -c + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c "pip3 install capstone keystone-engine z3-solver; python3 /panda/panda/python/examples/unicorn/taint_sym_x86_64.py; if [ $? -eq 0 ]; then echo -e 'TEST PASSED!' && exit 0; else echo 'TEST FAILED!' && exit 1; fi" - make_check: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - # See output from `make check-help`: we're just splitting `make check` into all the things it does - # so we can run them in parallel: arch-specific qtests, plus a few others - target: [check-qtest-x86_64, check-qtest-i386, check-qtest-arm, check-qtest-mips, check-qtest-mipsel, check-qtest-ppc, check-block, check-unit, check-qapi-schema] - - steps: - - name: Run Individual QEMU tests + - name: Run PyPanda Tests + if: matrix.test_type == 'pypanda' run: >- - docker run --name panda_test_${{ matrix.target }}_${GITHUB_RUN_ID} + wget -q https://panda-re.mit.edu/qcows/linux/ubuntu/1604/x86/ubuntu_1604_x86.qcow; + docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID} + --mount type=bind,source=$(pwd)/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow -e PANDA_TEST=yes --cap-add SYS_NICE - --rm -t "panda_local_${{ github.sha }}" bash -c - "cd /panda/build && make ${{ matrix.target }}" - - pypanda_tests: - if: github.repository == 'panda-re/panda' - runs-on: self-hosted - needs: [build_container] - - strategy: - matrix: - # See output from `make check-help`: we're just splitting `make check` into all the things it does - # so we can run them in parallel: arch-specific qtests, plus a few others - test_script: [dyn_hooks, copy_test, file_fake, file_hook, generic_tests, monitor_cmds, multi_proc_cbs, sleep_in_cb, syscalls, record_no_snap, sig_suppress] + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c + "cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && python3 ${{ matrix.test_script }}.py" - steps: - - name: Run individual pypanda tests - # TODO: pip requirements install here should be moved to Docker image build to save test time + - name: Run make Tests + if: matrix.test_type == 'make_check' run: >- docker run --name panda_test_${{ matrix.test_script }}_${GITHUB_RUN_ID} - --mount type=bind,source=/home/panda/regdir/qcows/ubuntu_1604_x86.qcow,target=/root/.panda/ubuntu_1604_x86.qcow -e PANDA_TEST=yes --cap-add SYS_NICE - --rm -t "panda_local_${{ github.sha }}" bash -c - "cd /panda/panda/python/tests/ && make && pip3 install -r requirements.txt && python3 ${{ matrix.test_script }}.py" + --rm -t "ghcr.io/${{ github.repository_owner }}/panda_local:${{ github.sha }}" bash -c + "cd /panda/build && make ${{ matrix.test_script }}" cleanup: # Cleanup after prior jobs finish - even if they fail - needs: [taint_tests, sym_trace_tests, make_check, pypanda_tests] - runs-on: self-hosted + needs: [tests] + runs-on: panda-arc if: always() steps: @@ -143,9 +184,9 @@ jobs: docker image prune --all -f --filter "until=72h" docker builder prune -af --filter "until=72h" - build_and_check_fork: # Forked repos can't use self-hosted test suite - just checkout and run make check + build_and_check_fork: # Forked repos can't use panda-arc test suite - just checkout and run make check if: github.repository != 'panda-re/panda' - runs-on: ubuntu-latest + runs-on: panda-arc steps: - uses: actions/checkout@v1 # Clones code into to /home/runner/work/panda diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 5fdafc34bc7..1148f73755d 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -9,24 +9,27 @@ on: jobs: build_dev: if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/dev' - runs-on: self-hosted + runs-on: panda-arc steps: - name: Checkout PANDA at current commit uses: actions/checkout@v2 - name: Docker login run: docker login -u pandare -p ${{secrets.pandare_dockerhub}} - - - name: Build Bionic container - # Push both dev and regular container - run: DOCKER_BUILDKIT=1 docker build --progress=plain --target=panda -t pandare/panda:${GITHUB_SHA} $GITHUB_WORKSPACE; - docker tag pandare/panda:${GITHUB_SHA} pandare/panda:latest; - docker push pandare/panda:${GITHUB_SHA}; - docker push pandare/panda; - DOCKER_BUILDKIT=1 docker build --progress=plain --target=developer -t pandare/pandadev:${GITHUB_SHA} $GITHUB_WORKSPACE; - docker tag pandare/pandadev:${GITHUB_SHA} pandare/pandadev:latest; - docker push pandare/pandadev:${GITHUB_SHA}; - docker push pandare/pandadev; + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build panda:latest + uses: docker/build-push-action@v5 + with: + tags: pandare/panda:${GITHUB_SHA} + target: panda + - name: Build pandadev:latest + uses: docker/build-push-action@v5 + with: + tags: pandare/pandadev:${GITHUB_SHA} + target: developer - name: Checkout docs and reset run: rm -rf "${GITHUB_WORKSPACE}/auto_pydoc"; @@ -55,7 +58,7 @@ jobs: build_stable: if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/stable' - runs-on: self-hosted + runs-on: panda-arc steps: - name: Checkout PANDA at current commit uses: actions/checkout@v1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4863a746a46..d2425d5a088 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ on: jobs: stale: - runs-on: ubuntu-latest + runs-on: panda-arc permissions: issues: write pull-requests: write diff --git a/.gitmodules b/.gitmodules index beb2b4fb3b4..aa8c2e6edb3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,36 +1,36 @@ [submodule "roms/vgabios"] path = roms/vgabios - url = https://git.qemu.org/git/vgabios.git + url = https://gitlab.com/qemu-project/vgabios.git [submodule "roms/seabios"] path = roms/seabios - url = https://git.qemu.org/git/seabios.git + url = https://gitlab.com/qemu-project/seabios.git [submodule "roms/SLOF"] path = roms/SLOF - url = https://git.qemu.org/git/SLOF.git + url = https://gitlab.com/qemu-project/SLOF.git [submodule "roms/ipxe"] path = roms/ipxe - url = https://git.qemu.org/git/ipxe.git + url = https://gitlab.com/qemu-project/ipxe.git [submodule "roms/openbios"] path = roms/openbios - url = https://git.qemu.org/git/openbios.git + url = https://gitlab.com/qemu-project/openbios.git [submodule "roms/openhackware"] path = roms/openhackware - url = https://git.qemu.org/git/openhackware.git + url = https://gitlab.com/qemu-project/openhackware.git [submodule "roms/qemu-palcode"] path = roms/qemu-palcode url = https://github.com/rth7680/qemu-palcode.git [submodule "roms/sgabios"] path = roms/sgabios - url = https://git.qemu.org/git/sgabios.git + url = https://gitlab.com/qemu-project/sgabios.git [submodule "pixman"] path = pixman - url = https://anongit.freedesktop.org/git/pixman + url = https://github.com/coolkingcole/pixman.git [submodule "dtc"] path = dtc - url = https://git.qemu.org/git/dtc.git + url = https://github.com/qemu/dtc.git [submodule "roms/u-boot"] path = roms/u-boot - url = https://git.qemu.org/git/u-boot.git + url = https://gitlab.com/qemu-project/u-boot.git [submodule "roms/skiboot"] path = roms/skiboot - url = https://git.qemu.org/git/skiboot.git + url = https://gitlab.com/qemu-project/skiboot.git diff --git a/Dockerfile b/Dockerfile index 86f243d09a2..36b08f1a2b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,8 +61,9 @@ RUN git -C /panda submodule update --init dtc && \ --target-list="${TARGET_LIST}" \ --prefix=/usr/local \ --disable-numa \ - --enable-llvm && \ - (make -C /panda/build -j "$(nproc)" || make) # If multi-core make fails, remake once to give a good error at the end + --enable-llvm + +RUN make -C /panda/build -j "$(nproc)" #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3 FROM builder as developer diff --git a/panda/python/core/requirements.txt b/panda/python/core/requirements.txt new file mode 100644 index 00000000000..b88df3026f7 --- /dev/null +++ b/panda/python/core/requirements.txt @@ -0,0 +1,3 @@ +cffi>=1.14.3 +protobuf==3.0.0 +colorama diff --git a/panda/scripts/install_ubuntu.sh b/panda/scripts/install_ubuntu.sh index 8cad2a52f63..d54c40a4898 100755 --- a/panda/scripts/install_ubuntu.sh +++ b/panda/scripts/install_ubuntu.sh @@ -110,7 +110,7 @@ if [[ !$(ldconfig -p | grep -q libcapstone.so.4) ]]; then echo "Installing libcapstone v4" pushd /tmp && \ curl -o /tmp/cap.tgz -L https://github.com/aquynh/capstone/archive/4.0.2.tar.gz && \ - tar xvf cap.tgz && cd capstone-4.0.2/ && ./make.sh && $SUDO make install && cd /tmp && \ + tar xvf cap.tgz && cd capstone-4.0.2/ && MAKE_JOBS=$(nproc) ./make.sh && $SUDO make install && cd /tmp && \ rm -rf /tmp/capstone-4.0.2 $SUDO ldconfig popd @@ -153,6 +153,7 @@ pushd build progress "PANDA is built and ready to use in panda/build/[arch]-softmmu/panda-system-[arch]." cd ../panda/python/core +$SUDO python3 -m pip install -r requirements.txt $SUDO python3 setup.py install python3 -c "import pandare; panda = pandare.Panda(generic='i386')" # Make sure it worked progress "Pypanda successfully installed"