Merge branch 'master' into master

Author: coderzc

.asf.yaml

@@ -39,13 +39,13 @@ github:
         strict: false
         # contexts are the names of checks that must pass
         contexts:
-          - computer-ci
+          #- computer-ci # always stuck in CI
           - check-license-header
           - Analyze (java)
       required_pull_request_reviews:
         dismiss_stale_reviews: true
         require_code_owner_reviews: false
-        required_approving_review_count: 2
+        required_approving_review_count: 1
 
 notifications:
   # use https://selfserve.apache.org to manage it

.gitattributes

@@ -6,4 +6,6 @@
 
 # ignored directory
 .github/ export-ignore
-computer-k8s-operator/crd-generate/bin/ export-ignore
+computer/computer-k8s-operator/crd-generate/bin/ export-ignore
+computer/computer-dist/scripts/ export-ignore
+computer/computer-dist/release-docs/ export-ignore

.github/workflows/ci.yml

@@ -16,10 +16,11 @@ jobs:
     env:
       USE_STAGE: 'true' # Whether to include the stage repository.
       TRAVIS_DIR: computer-dist/src/assembly/travis
-      KUBERNETES_VERSION: 1.20.1
       BSP_ETCD_URL: http://localhost:2579
-      # TODO: delete this env in the future (replaced by docker way now)
-      HUGEGRAPH_SERVER_COMMIT_ID: d01c8737d7d5909119671953521f1401dcd1a188
+      KUBERNETES_VERSION: 1.20.1
+      # TODO: adapt the HugeGraph Server/Loader version to 1.5.0 (EdgeID has 5 parts now)
+      # NOTE: Remember to adaptor/update the version before new release
+      GRAPH_ENV_VERSION: 1.3.0
 
     steps:
       - name: Checkout
@@ -78,8 +79,7 @@ jobs:
       - name: Prepare env and service
         run: |
           $TRAVIS_DIR/install-env.sh
-          $TRAVIS_DIR/load-data-into-hugegraph.sh
-          #$TRAVIS_DIR/install-hugegraph-from-source.sh $HUGEGRAPH_SERVER_COMMIT_ID
+          $TRAVIS_DIR/load-data-into-hugegraph.sh ${{ env.GRAPH_ENV_VERSION }}
 
       - name: Install JDK 11
         uses: actions/setup-java@v3

NOTICE

@@ -1,5 +1,5 @@
 Apache HugeGraph(incubating)
-Copyright 2022-2023 The Apache Software Foundation
+Copyright 2022-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -5,16 +5,20 @@
 [![codecov](https://codecov.io/gh/apache/incubator-hugegraph-computer/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/incubator-hugegraph-computer)
 [![Docker Pulls](https://img.shields.io/docker/pulls/hugegraph/hugegraph-computer)](https://hub.docker.com/repository/docker/hugegraph/hugegraph-computer)
 
-The [hugegraph-computer](./computer/README.md) is a distributed graph processing system for hugegraph.
+The [hugegraph-computer](./computer/README.md) is a distributed graph processing system for hugegraph. 
+(Also, the in-memory computing engine(vermeer) is on the way 🚧)
 
 ## Learn More
 
-The [project homepage](https://hugegraph.apache.org/docs/) contains more information about hugegraph-computer.
+The [project homepage](https://hugegraph.apache.org/docs/quickstart/hugegraph-computer/) contains more information about hugegraph-computer.
 
 And here are links of other repositories:
-1. [hugegraph-server](https://github.com/apache/hugegraph) (graph's core component - OLTP server)
-2. [hugegraph-toolchain](https://github.com/apache/hugegraph-toolchain) (include loader/dashboard/tool/client)
-3. [hugegraph-website](https://github.com/apache/hugegraph-doc) (include doc & website code)
+
+1. [hugegraph](https://github.com/apache/hugegraph) (graph's core component - Graph server + PD + Store)
+2. [hugegraph-toolchain](https://github.com/apache/hugegraph-toolchain) (graph tools **[loader](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-loader)/[dashboard](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-hubble)/[tool](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-tools)/[client](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-client)**)
+3. [hugegraph-ai](https://github.com/apache/incubator-hugegraph-ai) (integrated **Graph AI/LLM/KG** system)
+4. [hugegraph-website](https://github.com/apache/hugegraph-doc) (**doc & website** code)
+
 
 ## Note
 
@@ -40,5 +44,5 @@ hugegraph-computer is licensed under [Apache 2.0](https://github.com/apache/incu
  - Feedback Email: [dev@hugegraph.apache.org](mailto:dev@hugegraph.apache.org) ([subscriber](https://hugegraph.apache.org/docs/contribution-guidelines/subscribe/) only)
  - WeChat public account: Apache HugeGraph, welcome to scan this QR code to follow us.
 
- <img src="https://github.com/apache/incubator-hugegraph-doc/blob/master/assets/images/wechat.png?raw=true" alt="QR png" width="350"/>
+ <img src="https://github.com/apache/hugegraph-doc/blob/master/assets/images/wechat.png?raw=true" alt="QR png" width="350"/>
 

computer/README.md

@@ -5,8 +5,8 @@ The hugegraph-computer is a distributed graph processing system for hugegraph. I
 ## Features
 
 - Support distributed MPP graph computing, and integrates with HugeGraph as graph input/output storage.
-- Based on BSP(Bulk Synchronous Parallel) model, an algorithm performs computing through multiple parallel iterations, every iteration is a superstep.
+- Based on BSP (Bulk Synchronous Parallel) model, an algorithm performs computing through multiple parallel iterations, every iteration is a superstep.
 - Auto memory management. The framework will never be OOM(Out of Memory) since it will split some data to disk if it doesn't have enough memory to hold all the data.
-- The part of edges or the messages of super node can be in memory, so you will never lose it.
+- The part of edges or the messages of supernode can be in memory, so you will never lose it.
 - You can load the data from HDFS or HugeGraph, output the results to HDFS or HugeGraph, or adapt any other systems manually as needed.
-- Easy to develop a new algorithm. You just need to focus on a vertex only processing just like as in a single server, without worrying about message transfer and memory/storage management.
+- Easy to develop a new algorithm. You need to focus on a vertex only processing just like as in a single server, without worrying about message transfer and memory/storage management.

computer/computer-dist/scripts/apache-release.sh

@@ -39,9 +39,10 @@ echo "In the work dir: $(pwd)"
 rm -rf dist && mkdir -p dist/apache-${REPO}
 
 # step1: package the source code
-cd ../../ || exit
+cd ../../../ || exit
+pwd
 git archive --format=tar.gz \
-  --output="computer-dist/scripts/dist/apache-${REPO}/apache-${REPO}-incubating-${RELEASE_VERSION}-src.tar.gz" \
+  --output="computer/computer-dist/scripts/dist/apache-${REPO}/apache-${REPO}-incubating-${RELEASE_VERSION}-src.tar.gz" \
   --prefix="apache-${REPO}-incubating-${RELEASE_VERSION}-src/" "${GIT_BRANCH}" || exit
 cd - || exit
 

computer/computer-dist/scripts/dependency/known-dependencies.txt

@@ -36,14 +36,14 @@ commons-logging-1.1.3.jar
 commons-math3-3.1.1.jar
 commons-net-3.6.jar
 commons-text-1.9.jar
-computer-algorithm-1.3.0.jar
-computer-api-1.3.0.jar
-computer-core-1.3.0.jar
-computer-dist-1.3.0.jar
-computer-driver-1.3.0.jar
-computer-k8s-1.3.0.jar
-computer-k8s-operator-1.3.0.jar
-computer-yarn-1.3.0.jar
+computer-algorithm-1.5.0.jar
+computer-api-1.5.0.jar
+computer-core-1.5.0.jar
+computer-dist-1.5.0.jar
+computer-driver-1.5.0.jar
+computer-k8s-1.5.0.jar
+computer-k8s-operator-1.5.0.jar
+computer-yarn-1.5.0.jar
 curator-client-2.13.0.jar
 curator-framework-2.13.0.jar
 curator-recipes-2.13.0.jar

computer/computer-dist/scripts/dependency/regenerate_known_dependencies.sh

@@ -16,7 +16,7 @@
 # under the License.
 #
 
-BASE_PATH=$(cd $(dirname $0); pwd)
+BASE_PATH=$(cd $(dirname $0) || exit; pwd)
 DEP_PATH=$BASE_PATH/all_dependencies
 FILE_NAME=${1:-known-dependencies.txt}
 
@@ -25,9 +25,9 @@ if [[ -d $DEP_PATH ]];then
   rm -r -f $DEP_PATH
 fi
 
-cd $BASE_PATH/../../../
+cd $BASE_PATH/../../../ || exit
 
-mvn dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=$DEP_PATH
+mvn dependency:copy-dependencies -DincludeScope=runtime -DoutputDirectory=$DEP_PATH -P stage
 
 ls $DEP_PATH | egrep -v "^hugegraph" | sort -n > $BASE_PATH/$FILE_NAME
 rm -r -f $DEP_PATH

computer/computer-dist/src/assembly/travis/install-hugegraph-from-source.sh

@@ -27,7 +27,7 @@ fi
 COMMIT_ID=$1
 HUGEGRAPH_GIT_URL="https://github.com/apache/hugegraph.git"
 
-git clone --depth 100 ${HUGEGRAPH_GIT_URL} hugegraph
+git clone --depth 300 ${HUGEGRAPH_GIT_URL} hugegraph
 cd hugegraph
 git checkout "${COMMIT_ID}"
 mvn package -DskipTests -ntp

computer/computer-dist/src/assembly/travis/load-data-into-hugegraph.sh

@@ -23,14 +23,14 @@ DATASET_DIR=${TRAVIS_DIR}/../dataset
 
 docker network create ci
 # Note: we need wait for server start finished, so start it first
-docker run -itd --name=graph --network ci -p 8080:8080 hugegraph/hugegraph:latest && sleep 6
+docker run -itd --name=graph --network ci -p 8080:8080 hugegraph/hugegraph:"${GRAPH_ENV_VERSION}" && sleep 6
 
 wget http://files.grouplens.org/datasets/movielens/ml-latest-small.zip
-unzip -d ${DATASET_DIR} ml-latest-small.zip
+unzip -d "${DATASET_DIR}" ml-latest-small.zip
 
-cd ${DATASET_DIR}/.. && pwd && ls -lh *
+cd "${DATASET_DIR}"/.. && pwd && ls -lh ./*
 
-docker run -id --name=loader --network ci hugegraph/loader:latest
+docker run -id --name=loader --network ci hugegraph/loader:"${GRAPH_ENV_VERSION}"
 docker cp dataset loader:/dataset || exit 1
 
 docker exec -i loader ls -lh /dataset

computer/pom.xml

@@ -91,12 +91,11 @@
     </prerequisites>
 
     <properties>
-        <!-- TODO: update the version after toolchain v1.2 fixed -->
-        <revision>1.3.0</revision>
-        <hugegraph-common-version>1.3.0</hugegraph-common-version>
-        <hugegraph-client-version>1.3.0</hugegraph-client-version>
-        <hugegraph-rpc-version>1.3.0</hugegraph-rpc-version>
-        <hugegraph-loader-version>1.3.0</hugegraph-loader-version>
+        <revision>1.5.0</revision>
+        <!-- TODO: fix deps conflicts & update to 1.5.0 -->
+        <hugegraph-commons-version>1.3.0</hugegraph-commons-version>
+        <!-- FIXME: DegreeCentralityTest:L50 failed with loader 1.5.0      -->
+        <hugegraph-toolchain-version>1.3.0</hugegraph-toolchain-version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <top.level.dir>${project.basedir}/..</top.level.dir>
         <release.name>hugegraph-computer</release.name>
@@ -128,12 +127,12 @@
             <dependency>
                 <groupId>org.apache.hugegraph</groupId>
                 <artifactId>hugegraph-common</artifactId>
-                <version>${hugegraph-common-version}</version>
+                <version>${hugegraph-commons-version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.hugegraph</groupId>
                 <artifactId>hugegraph-rpc</artifactId>
-                <version>${hugegraph-rpc-version}</version>
+                <version>${hugegraph-commons-version}</version>
                 <exclusions>
                     <!-- conflict with etcd -->
                     <exclusion>
@@ -154,7 +153,7 @@
             <dependency>
                 <groupId>org.apache.hugegraph</groupId>
                 <artifactId>hugegraph-client</artifactId>
-                <version>${hugegraph-client-version}</version>
+                <version>${hugegraph-toolchain-version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.hugegraph</groupId>
@@ -251,7 +250,7 @@
             <dependency>
                 <groupId>org.apache.hugegraph</groupId>
                 <artifactId>hugegraph-loader</artifactId>
-                <version>${hugegraph-loader-version}</version>
+                <version>${hugegraph-toolchain-version}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.apache.hive</groupId>