Consolidating all installations to requirements.txt and other minor

fixes

Author: AndrewQuijano

.github/workflows/parallel_tests.yml

@@ -20,9 +20,9 @@ on:
 jobs:
 
   test_installer: # test install_ubuntu.sh
-    runs-on: panda-arc # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet)
+    runs-on: panda-arc
     container:
-      image: ubuntu:20.04
+      image: ubuntu:22.04
     steps:
     - name: Update
       run: apt-get -qq update -y
@@ -31,9 +31,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: '3.10'
     - name: Install Python dev headers
-      run: apt-get -qq install -y libpython3.9-dev
+      run: apt-get -qq install -y libpython3.10-dev
     - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
     - name: Lint PyPANDA with flake8
       run: |

Dockerfile

@@ -1,10 +1,11 @@
 ARG BASE_IMAGE="ubuntu:20.04"
 ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,aarch64-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu,mips64-softmmu,mips64el-softmmu"
-ARG LIBOSI_VERSION="v0.1.7"
+ARG INSTALL_PREFIX="/usr/local/"
 
 ### BASE IMAGE
-FROM $BASE_IMAGE as base
+FROM $BASE_IMAGE AS base
 ARG BASE_IMAGE
+ARG INSTALL_PREFIX
 
 # Copy dependencies lists into container. We copy them all and then do a mv because
 # we need to transform base_image into a windows compatible filename which we can't
@@ -16,22 +17,19 @@ RUN mv /tmp/$(echo "$BASE_IMAGE" | sed 's/:/_/g')_build.txt /tmp/build_dep.txt &
 # Base image just needs runtime dependencies
 RUN [ -e /tmp/base_dep.txt ] && \
     apt-get -qq update && \
-    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
+    DEBIAN_FRONTEND=noninteractive apt-get -qq install -y --no-install-recommends curl jq $(cat /tmp/base_dep.txt | grep -o '^[^#]*') && \
     apt-get clean
 
 ### BUILD IMAGE - STAGE 2
 FROM base AS builder
 ARG BASE_IMAGE
 ARG TARGET_LIST
-ARG LIBOSI_VERSION
+ARG INSTALL_PREFIX
 
 RUN [ -e /tmp/build_dep.txt ] && \
     apt-get -qq update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $(cat /tmp/build_dep.txt | grep -o '^[^#]*') && \
     apt-get clean && \
-    python3 -m pip install --upgrade --no-cache-dir pip && \
-    python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \
-    python3 -m pip install --upgrade --no-cache-dir "capstone" && \
     curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal
 
 # Then install capstone from source
@@ -43,24 +41,31 @@ RUN cd /tmp && \
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # install libosi
-RUN cd /tmp && curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb && dpkg -i /tmp/libosi_$(echo "$BASE_IMAGE" | awk -F':' '{print $2}').deb
+RUN cd /tmp && \
+    BASE_IMAGE_VERSION=$(echo "$BASE_IMAGE" | awk -F':' '{print $2}') && \
+    LIBOSI_VERSION=$(curl -s https://api.github.com/repos/panda-re/libosi/releases/latest | jq -r .tag_name) && \
+    curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_${BASE_IMAGE_VERSION}.deb && \
+    dpkg -i /tmp/libosi_${BASE_IMAGE_VERSION}.deb && \
+    rm -rf /tmp/libosi_${BASE_IMAGE_VERSION}.deb
 
 # Build and install panda
 # Copy repo root directory to /panda, note we explicitly copy in .git directory
 # Note .dockerignore file keeps us from copying things we don't need
+# PyPANDA needs CFFI from pip (the version in apt is too old)
 COPY . /panda/
 COPY .git /panda/
+RUN pip install -r /panda/panda/python/core/requirements.txt
 
 # Note we diable NUMA for docker builds because it causes make check to fail in docker
 RUN git -C /panda submodule update --init dtc && \
-    git -C /panda rev-parse HEAD > /usr/local/panda_commit_hash && \
+    git -C /panda rev-parse HEAD > ${INSTALL_PREFIX}panda_commit_hash && \
     mkdir  /panda/build && cd /panda/build && \
     python3 -m pip install setuptools_scm && \
     python3 -m pip install build && \
     python3 -m setuptools_scm -r .. --strip-dev 2>/dev/null >/tmp/savedversion && \
     /panda/configure \
         --target-list="${TARGET_LIST}" \
-        --prefix=/usr/local \
+        --prefix=${INSTALL_PREFIX} \
         --disable-numa \
         --enable-llvm && \
     rm -rf /panda/.git
@@ -69,7 +74,8 @@ RUN git -C /panda submodule update --init dtc && \
 RUN PRETEND_VERSION=$(cat /tmp/savedversion) make -C /panda/build -j "$(nproc)"
 
 #### Develop setup: panda built + pypanda installed (in develop mode) - Stage 3
-FROM builder as developer
+FROM builder AS developer
+ARG INSTALL_PREFIX
 RUN cd /panda/panda/python/core && \
     python3 create_panda_datatypes.py && \
     PRETEND_VERSION=$(cat /tmp/savedversion) pip install -e . && \
@@ -82,22 +88,19 @@ RUN cd /panda/panda/python/core && \
 WORKDIR /panda/
 
 #### Install PANDA + pypanda from builder - Stage 4
-FROM builder as installer
+FROM builder AS installer
+ARG INSTALL_PREFIX
 RUN  make -C /panda/build install && \
-    rm -r /usr/local/lib/panda/*/cosi \
-        /usr/local/lib/panda/*/cosi_strace \
-        /usr/local/lib/panda/*/gdb \
-        /usr/local/lib/panda/*/snake_hook \
-        /usr/local/lib/panda/*/rust_skeleton
+    rm -r ${INSTALL_PREFIX}lib/panda/*/cosi \
+        ${INSTALL_PREFIX}lib/panda/*/cosi_strace \
+        ${INSTALL_PREFIX}lib/panda/*/gdb \
+        ${INSTALL_PREFIX}lib/panda/*/snake_hook \
+        ${INSTALL_PREFIX}lib/panda/*/rust_skeleton
 
-# Install pypanda
-RUN cd /panda/panda/python/core && \
-    python3 create_panda_datatypes.py --install && \
-    PRETEND_VERSION=$(cat /tmp/savedversion) pip install .
-RUN python3 -m pip install --ignore-install pycparser && python3 -m pip install --force-reinstall --no-binary :all: cffi
-# Build a whl too
+# Install pypanda and build wheel
 RUN cd /panda/panda/python/core && \
     python3 create_panda_datatypes.py --install && \
+    PRETEND_VERSION=$(cat /tmp/savedversion) pip install . && \
     PRETEND_VERSION=$(cat /tmp/savedversion) python3 -m build --wheel .
 
 # BUG: PANDA sometimes fails to generate all the necessary files for PyPANDA. This is a temporary fix to detect and fail when this occurs
@@ -106,41 +109,53 @@ RUN bash -c "ls $(pip show pandare | grep Location: | awk '{print $2}')/pandare/
 
 # this layer is used to strip shared objects and change python data to be
 # symlinks to the installed panda data directory
-FROM installer as cleanup
-RUN find /usr/local/lib/panda -name "*.so" -exec strip {} \;
+FROM installer AS cleanup
+ARG INSTALL_PREFIX
+RUN find ${INSTALL_PREFIX}lib/panda -name "*.so" -exec strip {} \;
 RUN PKG=`pip show pandare | grep Location: | awk '{print $2}'`/pandare/data; \
-    rm -rf $PKG/pc-bios && ln -s /usr/local/share/panda $PKG/pc-bios; \
+    rm -rf $PKG/pc-bios && ln -s ${INSTALL_PREFIX}share/panda $PKG/pc-bios; \
     for arch in `find $PKG -name "*-softmmu" -type d -exec basename {} \;` ; do \
         ARCHP=$PKG/$arch; \
         SARCH=`echo $arch | cut -d'-' -f 1`; \
         rm $ARCHP/libpanda-$SARCH.so $ARCHP/llvm-helpers-$SARCH.bc; \
-        ln -s /usr/local/share/panda/llvm-helpers-$SARCH.bc $ARCHP/llvm-helpers-$SARCH.bc1; \
-        ln -s /usr/local/bin/libpanda-$SARCH.so $ARCHP/libpanda-$SARCH.so; \ 
+        ln -s ${INSTALL_PREFIX}share/panda/llvm-helpers-$SARCH.bc $ARCHP/llvm-helpers-$SARCH.bc1; \
+        ln -s ${INSTALL_PREFIX}bin/libpanda-$SARCH.so $ARCHP/libpanda-$SARCH.so; \ 
         rm -rf $ARCHP/panda/plugins; \
-        ln -s /usr/local/lib/panda/$SARCH/ $ARCHP/panda/plugins; \
+        ln -s ${INSTALL_PREFIX}lib/panda/$SARCH/ $ARCHP/panda/plugins; \
     done
 
 ### Copy files for panda+pypanda from installer  - Stage 5
-FROM base as panda
+FROM base AS panda
+ARG INSTALL_PREFIX
+ARG TARGET_LIST
 
 # Include dependency lists for packager
 COPY --from=base /tmp/base_dep.txt /tmp
 COPY --from=base /tmp/build_dep.txt /tmp
 
 # Copy panda + libcapstone.so* + libosi libraries
-COPY --from=cleanup /usr/local /usr/local
+COPY --from=cleanup ${INSTALL_PREFIX} ${INSTALL_PREFIX}
 COPY --from=cleanup /usr/lib/libcapstone* /usr/lib/
-COPY --from=cleanup /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/
+# TODO: Once PR, https://github.com/panda-re/libosi/pull/17 is in, libosi installs to /usr/lib/x86_64-linux-gnu instead of /usr/lib
+COPY --from=cleanup /usr/lib/libosi.so /usr/lib/libiohal.so /usr/lib/liboffset.so /usr/lib/x86_64-linux-gnu/
 
 # Workaround issue #901 - ensure LD_LIBRARY_PATH contains the panda plugins directories
-#ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu"
-ENV LD_LIBRARY_PATH /usr/local/lib/python3.8/dist-packages/pandare/data/x86_64-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/i386-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/arm-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/ppc-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mips-softmmu/panda/plugins/:/usr/local/lib/python3.8/dist-packages/pandare/data/mipsel-softmmu/panda/plugins/
-#PANDA_PATH is used by rust plugins
-ENV PANDA_PATH /usr/local/lib/python3.8/dist-packages/pandare/data
-
+RUN LD_LIBRARY_PATH="" && \
+    for arch in $(echo $TARGET_LIST | tr ',' ' '); do \
+    if [ -z "$LD_LIBRARY_PATH" ]; then \
+        LD_LIBRARY_PATH="${INSTALL_PREFIX}lib/python3.8/dist-packages/pandare/data/${arch}/panda/plugins/"; \
+    else \
+        LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTALL_PREFIX}lib/python3.8/dist-packages/pandare/data/${arch}/panda/plugins/"; \
+    fi \
+    done && \
+    echo "${LD_LIBRARY_PATH}" > /tmp/ld_library_path
+ENV LD_LIBRARY_PATH $(cat /tmp/ld_library_path)
+
+# PANDA_PATH is used by rust plugins
+ENV PANDA_PATH ${INSTALL_PREFIX}lib/python3.8/dist-packages/pandare/data
 
 # Ensure runtime dependencies are installed for our libpanda objects and panda plugins
 RUN ldconfig && \
     update-alternatives --install /usr/bin/python python /usr/bin/python3 10 && \
-    if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \
-    if (ldd /usr/local/lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi
+    if (ldd ${INSTALL_PREFIX}lib/python*/dist-packages/pandare/data/*-softmmu/libpanda-*.so | grep 'not found'); then exit 1; fi && \
+    if (ldd ${INSTALL_PREFIX}lib/python*/dist-packages/pandare/data/*-softmmu/panda/plugins/*.so | grep 'not found'); then exit 1; fi

panda/debian/.gitignore

@@ -1 +1,2 @@
-panda.deb
+*.deb
+*.whl

panda/dependencies/ubuntu_18.04_base.txt

@@ -18,7 +18,7 @@ unzip
 # pyperipheral (only needed for armel)
 libpython3-dev
 
-# pypanda dependencies
+# panda python dependencies
 genisoimage
 libffi-dev
 python3-protobuf

panda/dependencies/ubuntu_20.04_base.txt

@@ -17,7 +17,7 @@ wget
 # pyperipheral (only needed for armel)
 libpython3-dev
 
-# pypanda dependencies
+# panda python dependencies
 genisoimage
 libffi-dev
 python3-protobuf

panda/dependencies/ubuntu_22.04_base.txt

@@ -17,7 +17,7 @@ wget
 # pyperipheral (only needed for armel)
 libpython3-dev
 
-# pypanda dependencies
+# panda python dependencies
 genisoimage
 libffi-dev
 python3-protobuf

panda/plugins/syscalls2/scripts/requirements2.txt

@@ -1,2 +1,2 @@
-Jinja2==3.1.2
+Jinja2>=3.1.4
 MarkupSafe==2.1.3

panda/python/core/.gitignore

@@ -8,3 +8,4 @@ data
 a
 __pycache__
 *.egg-info
+.eggs/

panda/python/core/pyproject.toml

@@ -34,7 +34,8 @@ requires-python = ">=3.6"
 dependencies = [
     "cffi>=1.14.3",
     "protobuf>=4.25.1",
-    "colorama"
+    "colorama",
+    "capstone==5.0.3"
 ]
 
 [tool.setuptools_scm]

panda/python/core/requirements.txt

@@ -1,3 +1,4 @@
 cffi>=1.14.3
 protobuf>=4.25.1
 colorama
+capstone==5.0.3

panda/scripts/install_ubuntu.sh

@@ -23,9 +23,15 @@ fi
 # Note package names should be consistent across Ubuntu versions.
 lsb_release --help &>/dev/null || $SUDO apt-get update -qq && $SUDO apt-get -qq install -y --no-install-recommends lsb-release
 git --help &>/dev/null || $SUDO apt-get -qq update && $SUDO apt-get -qq install -y --no-install-recommends git
+$SUDO apt-get install -y --no-install-recommends curl jq
 
 # some globals
-LIBOSI_VERSION="0.1.7"
+# TODO: Why is curl -s failing in panda-arc? For now, I'll put a fallback version
+LIBOSI_VERSION=$(curl -s https://api.github.com/repos/panda-re/libosi/releases/latest | jq -r .tag_name)
+if [ -z "$LIBOSI_VERSION" ]; then
+  # TODO: Update this, once this PR is in, https://github.com/panda-re/libosi/pull/17
+  LIBOSI_VERSION="v0.1.9"
+fi
 UBUNTU_VERSION=$(lsb_release -r | awk '{print $2}')
 PANDA_GIT="https://github.com/panda-re/panda.git"
 
@@ -76,7 +82,6 @@ if [ $version -eq 18 ]; then
   $SUDO apt-get update
 fi
 
-
 # Dependencies are for a major version, but the filenames include minor versions
 # So take our major version, find the first match in dependencies directory and run with it.
 # This will give us "./panda/dependencies/ubuntu:20.04" where ubuntu:20.04_build.txt or 20.04_base.txt exists
@@ -120,17 +125,12 @@ fi
 # if the windows introspection library is not installed, clone and install
 if [[ !$(dpkg -l | grep -q libosi) ]]; then
   pushd /tmp
-  curl -LJO https://github.com/panda-re/libosi/releases/download/v${LIBOSI_VERSION}/libosi_${UBUNTU_VERSION}.deb 
+  curl -LJO https://github.com/panda-re/libosi/releases/download/${LIBOSI_VERSION}/libosi_${UBUNTU_VERSION}.deb
   $SUDO dpkg -i /tmp/libosi_${UBUNTU_VERSION}.deb
   rm -rf /tmp/libosi_${UBUNTU_VERSION}.deb
   popd
 fi
 
-# PyPANDA needs CFFI from pip (the version in apt is too old)
-# Install system-wide since PyPANDA install will also be system-wide
-$SUDO python3 -m pip install pip
-$SUDO python3 -m pip install "cffi>1.14.3"
-
 progress "Trying to update DTC submodule"
 git submodule update --init dtc || true
 
@@ -139,6 +139,10 @@ if [ -d "build" ]; then
   rm -rf "build"
 fi
 
+# PyPANDA needs CFFI from pip (the version in apt is too old)
+# Install system-wide since PyPANDA install will also be system-wide
+$SUDO pip install -r ./panda/python/core/requirements.txt
+
 progress "Building PANDA..."
 mkdir build
 pushd build