Finish Python updates, and update the Debian packaging to be more

standardized

Author: AndrewQuijano

.github/workflows/parallel_tests.yml

@@ -20,9 +20,9 @@ on:
 jobs:
 
   test_installer: # test install_ubuntu.sh
-    runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet)
+    runs-on: panda-arc
     container:
-      image: ubuntu:20.04
+      image: ubuntu:22.04
     steps:
     - name: Update
       run: apt-get -qq update -y
@@ -31,9 +31,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: Install Python dev headers
-      run: apt-get -qq install -y libpython3.9-dev
+      run: apt-get -qq install -y libpython3.10-dev
     - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
     - name: Lint PyPANDA with flake8
       run: |

.github/workflows/publish_docker.yml

@@ -46,7 +46,7 @@ jobs:
 
       - name: Build package
         working-directory: panda/debian
-        run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }}
+        run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }} ${{ needs.create_release.outputs.v-version }}
 
       - name: Upload wheel and debian packages to release
         uses: softprops/action-gh-release@v2

Dockerfile

@@ -1,9 +1,8 @@
 ARG BASE_IMAGE="ubuntu:20.04"
 ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu,mips64-softmmu,mips64el-softmmu"
-ARG LIBOSI_VERSION="v0.1.7"
 
 ### BASE IMAGE
-FROM $BASE_IMAGE as base
+FROM $BASE_IMAGE AS base
 ARG BASE_IMAGE
 
 # Copy dependencies lists into container. We copy them all and then do a mv because
@@ -16,22 +15,19 @@ RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt &
 # Base image just needs runtime dependencies
 RUN [ -e /tmp/base_dep.txt ] && \
     apt-get -qq update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
+    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl jq $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
     apt-get clean
 
 ### BUILD IMAGE - STAGE 2
 FROM base AS builder
 ARG BASE_IMAGE
 ARG TARGET_LIST
-ARG LIBOSI_VERSION
 
 RUN [ -e /tmp/build_dep.txt ] && \
     apt-get -qq update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \
     apt-get clean && \
     python3 -m pip install --upgrade --no-cache-dir pip && \
-    python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \
-    python3 -m pip install --upgrade --no-cache-dir "capstone" && \
     curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal
 
 # Then install capstone from source
@@ -43,13 +39,19 @@ RUN cd /tmp && \
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # install libosi
-RUN cd /tmp && curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb && dpkg -i /tmp/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb
+RUN cd /tmp && \
+    BASE_IMAGE_VERSION=$(echo "$BASE_IMAGE" | awk -F':' '{print $2}') && \
+    LIBOSI_VERSION=$(curl -s https://api.github.com/repos/panda-re/libosi/releases/latest | jq -r .tag_name) && \
+    curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_${BASE_IMAGE_VERSION}.deb && \
+    dpkg -i /tmp/libosi_${BASE_IMAGE_VERSION}.deb && \
+    rm -rf /tmp/libosi_${BASE_IMAGE_VERSION}.deb
 
 # Build and install panda
 # Copy repo root directory to /panda, note we explicitly copy in .git directory
 # Note .dockerignore file keeps us from copying things we don't need
 COPY . /panda/
 COPY .git /panda/
+RUN python3 -m pip install -r /panda/panda/python/core/requirements.txt
 
 # Note we diable NUMA for docker builds because it causes make check to fail in docker
 RUN git -C /panda submodule update --init dtc && \
@@ -64,12 +66,11 @@ RUN git -C /panda submodule update --init dtc && \
         --disable-numa \
         --enable-llvm && \
     rm -rf /panda/.git
-    
 
 RUN PRETEND_VERSION=$(cat /tmp/savedversion) make -C /panda/build -j "$(nproc)"
 
 #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3
-FROM builder as developer
+FROM builder AS developer
 RUN cd /panda/panda/python/core && \
     python3 create_panda_datatypes.py && \
     PRETEND_VERSION=$(cat /tmp/savedversion) pip install -e . && \
@@ -82,33 +83,27 @@ RUN cd /panda/panda/python/core && \
 WORKDIR /panda/
 
 #### Install PANDA + pypanda from builder - Stage 4
-FROM builder as installer
+FROM builder AS installer
 RUN  make -C /panda/build install && \
     rm -r /usr/local/lib/panda/*/cosi \
         /usr/local/lib/panda/*/cosi_strace \
         /usr/local/lib/panda/*/gdb \
         /usr/local/lib/panda/*/snake_hook \
         /usr/local/lib/panda/*/rust_skeleton
 
-# Install pypanda
+# Build wheel and install pypanda
 RUN cd /panda/panda/python/core && \
     python3 create_panda_datatypes.py --install && \
+    PRETEND_VERSION=$(cat /tmp/savedversion) python3 -m build --wheel . && \
     PRETEND_VERSION=$(cat /tmp/savedversion) pip install .
-RUN python3 -m pip install --upgrade pip "setuptools<65.6.0" && \
-    python3 -m pip install "pycparser<2.22" && \
-    python3 -m pip install --force-reinstall --no-binary :all: cffi
-# Build a whl too
-RUN cd /panda/panda/python/core && \
-    python3 create_panda_datatypes.py --install && \
-    PRETEND_VERSION=$(cat /tmp/savedversion) python3 -m build --wheel .
 
 # BUG: PANDA sometimes fails to generate all the necessary files for PyPANDA. This is a temporary fix to detect and fail when this occurs
 RUN ls -alt $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/
 RUN bash -c "ls $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/panda_{aarch64_64,arm_32,mips64_64,mips_32,mipsel_32,ppc_32,ppc_64,x86_64_64,i386_32}.py"
 
 # this layer is used to strip shared objects and change python data to be
 # symlinks to the installed panda data directory
-FROM installer as cleanup
+FROM installer AS cleanup
 RUN find /usr/local/lib/panda -name "*.so" -exec strip {} \;
 RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \
     rm -rf $PKG/pc-bios && ln -s /usr/local/share/panda $PKG/pc-bios; \
@@ -123,24 +118,23 @@ RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \
     done
 
 ### Copy files for panda+pypanda from installer  - Stage 5
-FROM base as panda
+FROM base AS panda
 
 # Include dependency lists for packager
 COPY --from=base /tmp/base_dep.txt /tmp
 COPY --from=base /tmp/build_dep.txt /tmp
 
 # Copy panda + libcapstone.so* + libosi libraries
 COPY --from=cleanup /usr/local /usr/local
-COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/
-COPY --from=cleanup /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/
+COPY --from=cleanup /usr/lib/libcapstone.so* /usr/lib/x86_64-linux-gnu/
+COPY --from=cleanup /usr/lib/x86_64-linux-gnu/libosi.so /usr/lib/x86_64-linux-gnu/libiohal.so /usr/lib/x86_64-linux-gnu/liboffset.so /usr/lib/x86_64-linux-gnu/
 
 # Workaround issue #901 - ensure LD_LIBRARY_PATH contains the panda plugins directories
 #ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu"
 ENV LD_LIBRARY_PATH /usr/local/lib/python3.8/dist-packages/pandare/data/x86_64-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/i386-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/arm-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/ppc-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mips-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mipsel-softmmu/panda/plugins/
 #PANDA_PATH is used by rust plugins
 ENV PANDA_PATH /usr/local/lib/python3.8/dist-packages/pandare/data
 
-
 # Ensure runtime dependencies are installed for our libpanda objects and panda plugins
 RUN ldconfig && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \

panda/debian/.gitignore

@@ -1 +1,2 @@
-panda.deb
+*.deb
+*.whl

panda/debian/Dockerfile

@@ -1,7 +1,9 @@
+ARG PACKAGE_VERSION=""
+
 # First run the main Dockerfile to build the base image and name it panda. Then we run here
 # to generate a debian package
 
-FROM debian:buster-slim
+FROM debian:bookworm-slim
 
 # Install necessary tools for packaging
 RUN apt-get -qq update && \
@@ -12,6 +14,9 @@ RUN apt-get -qq update && \
 COPY --from=panda /tmp/base_dep.txt /tmp
 COPY --from=panda /tmp/build_dep.txt /tmp
 
+# NOTE: If you use the panda debian package, you still need to manually curl 
+# the libosi and libcapstone-dev package as seen in the main Dockerfile/install_ubuntu.sh
+
 # Set up /package-root with files from panda we'll package
 COPY --from=panda /usr/local/bin/panda* /usr/local/bin/libpanda* /usr/local/bin/qemu-img /package-root/usr/local/bin/
 COPY --from=panda /usr/local/etc/panda /package-root/usr/local/etc/panda
@@ -21,6 +26,16 @@ COPY --from=panda /usr/local/share/panda /package-root/usr/local/share/panda
 # Create DEBIAN directory and control file
 COPY control /package-root/DEBIAN/control
 
+# Generate MD5 checksums for all files and save to DEBIAN/md5sums
+RUN cd /package-root && \
+    find . -type f ! -path './DEBIAN/*' -exec md5sum {} + | sed 's| \./| |' > /package-root/DEBIAN/md5sums
+
+# Update control file with the correct version, and place installed size
+ARG PACKAGE_VERSION
+RUN INSTALLED_SIZE=$(du -sk /package-root | cut -f1) && \
+    sed -i "s/^Installed-Size:.*/Installed-Size: ${INSTALLED_SIZE}/" /package-root/DEBIAN/control
+RUN sed -i "s/^Version:.*/Version: ${PACKAGE_VERSION}/" /package-root/DEBIAN/control
+
 # Update control file with dependencies
 # Build time. We only select dependencies that are not commented out or blank
 RUN dependencies=$(grep '^[a-zA-Z]' /tmp/build_dep.txt | tr '\n' ',' | sed 's/,,\+/,/g'| sed 's/,$//') && \
@@ -30,6 +45,9 @@ RUN dependencies=$(grep '^[a-zA-Z]' /tmp/build_dep.txt | tr '\n' ',' | sed 's/,,
 RUN dependencies=$(grep '^[a-zA-Z]' /tmp/base_dep.txt | tr '\n' ',' | sed 's/,,\+/,/g' | sed 's/,$//') && \
     sed -i "s/DEPENDS_LIST/Depends: ipxe-qemu,${dependencies}/" /package-root/DEBIAN/control
 
+# Add triggers script to run ldconfig after installation
+COPY triggers /package-root/DEBIAN/triggers
+
 # Build the package
 RUN fakeroot dpkg-deb --build /package-root /pandare.deb
 

panda/debian/control

@@ -1,9 +1,15 @@
 Package: pandare
-Version: 3.1.0
-Architecture: all
+Source: pandare
+Version: <version-placeholder>
+Architecture: amd64
 BUILD_DEPENDS_LIST
 DEPENDS_LIST
-Maintainer: Andrew Fasano <fasano@mit.edu>
+Maintainer: Luke Craig <luke.craig@mit.edu>
+Installed-Size: <size-in-kb>
+Section: devel
+Priority: optional
+Multi-Arch: same
+Homepage: https://github.com/panda-re/panda
 Description: dynamic analysis platform
  Platform for Architecture Neutral Dynamic Analysis (PANDA) is a processor
  emulator designed to support analyses of guest code. PANDA supports record-

panda/debian/setup.sh

@@ -25,36 +25,80 @@ if [[ $# -eq 1 ]]; then
 	echo "	To build a package for current Ubuntu version:"
 	echo "	  $0"
 	echo "	To build a package for a specific OS/version (only Ubuntu supported for now):"
-	echo "	  $0 <OS> <version>"
+	echo "	  $0 <OS> <ubuntu-version> <tag-version>"
 	exit 1
 fi
 
 if [[ $# -eq 2 ]]; then
 	version=$2
-
 else
 	version=$(lsb_release -r | awk '{print $2}')
 fi
 
+if [[ $# -eq 3 ]]; then
+	tag_version=$3
+else
+	tag_version='v3.1.0'
+fi
+
+# Remove leading 'v' if present, e. g. v1.5.1 -> 1.5.1
+if [[ "$tag_version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    tag_version=${tag_version:1}
+fi
+
+# Check if the version follows the format X.Y.Z, e. g. 1.5.1 or 1.9.1
+if [[ ! "$tag_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "ERROR: Version must be in the format X.Y.Z, provided tag version: $tag_version"
+    exit 1
+fi
+
 # Check if the given version is supported
 if [[ ! -f "../dependencies/ubuntu_${version}_base.txt" ]]; then
 	echo "ERROR: Ubuntu ${version} is not supported, no dependencies file found"
 	exit 1
 fi
 
+# Check if HTTP_PROXY and HTTPS_PROXY are set, if not set them to blank
+HTTP_PROXY="${HTTP_PROXY:-}"
+HTTPS_PROXY="${HTTPS_PROXY:-}"
+
 # Build the installer to generate the wheel file
-DOCKER_BUILDKIT=1 docker build --target installer -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../..
+DOCKER_BUILDKIT=1 docker build \
+    --target installer \
+    -t panda_installer \
+    --build-arg HTTP_PROXY="${HTTP_PROXY}" \
+    --build-arg HTTPS_PROXY="${HTTPS_PROXY}" \
+    --build-arg BASE_IMAGE="ubuntu:${version}" \
+    ../..
 
 # Copy wheel file out of container to host
-# this also preserves wheel name, which is important as pip install WILL fail if you arbitarily change the generated wheel file name
-docker run --rm -v $(pwd):/out panda bash -c "cp /panda/panda/python/core/dist/*.whl /out"
+# This also preserves wheel name, which is important as pip install WILL fail if you arbitrarily change the generated wheel file name
+docker run --rm \
+    -v $(pwd):/out \
+    panda_installer \
+    bash -c "cp /panda/panda/python/core/dist/*.whl /out"
 
 # Finish building main panda container for the target ubuntu version
-DOCKER_BUILDKIT=1 docker build --target panda -t panda --build-arg BASE_IMAGE="ubuntu:${version}" ../..
+DOCKER_BUILDKIT=1 docker build \
+    --cache-from panda_installer \
+    --target panda \
+    -t panda \
+    --build-arg BASE_IMAGE="ubuntu:${version}" \
+    ../..
 
 # Now build the packager container from that
-docker build -t packager .
+DOCKER_BUILDKIT=1 docker build \
+    --cache-from panda \
+    -t packager \
+    --build-arg HTTP_PROXY="${HTTP_PROXY}" \
+    --build-arg HTTPS_PROXY="${HTTPS_PROXY}" \
+    --build-arg PACKAGE_VERSION="${tag_version}" \
+    .
 
 # Copy deb file out of container to host
-docker run --rm -v $(pwd):/out packager bash -c "cp /pandare.deb /out"
-mv pandare.deb pandare_${version}.deb
+docker run --rm \
+    -v $(pwd):/out \
+    packager \
+    bash -c "cp /pandare.deb /out"
+
+mv pandare.deb pandare_${version}.deb

panda/debian/triggers

@@ -0,0 +1,2 @@
+# Trigger ldconfig after install
+activate-noawait ldconfig

panda/plugins/syscalls2/scripts/requirements2.txt

@@ -1,2 +1,2 @@
-Jinja2==3.1.2
+Jinja2>=3.1.4
 MarkupSafe==2.1.3

panda/python/core/.gitignore

@@ -8,3 +8,4 @@ data
 a
 __pycache__
 *.egg-info
+.eggs/

panda/python/core/create_panda_datatypes.py

@@ -690,14 +690,14 @@ def copy_objs():
                                           'and prep for PyPanda wheel installation')
     parser.add_argument('--install', '-i', dest='install', action='store_true',
                         help='If set, this means update pandare folder for installation')
-    parser.add_argument('--recompile', '-r', dest='recompile', action='store_true',
-                        help='If set, recompile the headers with cffi')
+    parser.add_argument('--no-compile', dest='compile', action='store_false',
+                        help='If set, do not compile the headers with cffi')
     args = parser.parse_args()
     """
     Install as a local module (not to system) by
         1) Creating datatype files for local-use
         2) Running regular setup tools logic
     """
-    main()
+    main(args.compile)
     if args.install:
         copy_objs()

panda/python/core/pyproject.toml

@@ -34,7 +34,8 @@ requires-python = ">=3.6"
 dependencies = [
     "cffi>=1.14.3",
     "protobuf>=4.25.1",
-    "colorama"
+    "colorama",
+    "capstone==5.0.3"
 ]
 
 [tool.setuptools_scm]

panda/python/core/requirements.txt

@@ -1,3 +1,4 @@
 cffi>=1.14.3
 protobuf>=4.25.1
+capstone==5.0.3
 colorama