Merge pull request #1857 from hyperledger/develop

Iroha v1.4-rc.2

Author: iceseer

.github/build-iroha1.src.yml

@@ -70,7 +70,6 @@ jobs:
   ## anyway please read .github/README.md
   check_workflow_yaml_coressponds_to_src_yaml:
     runs-on: ubuntu-20.04 #ubuntu-latest
-    #container: ubuntu:latest  ## This is required as barrier between AWS-hosted runners and GitHub-hosted runners - they have different set of software, so run in container
     name: Check if github workflows were properly made from sources
     steps:
       - &step_detect_commented_pr
@@ -232,14 +231,6 @@ jobs:
       matrix_dockerimage_release: ${{steps.matrixes.outputs.matrix_dockerimage_release}}
       matrix_dockerimage_debug:   ${{steps.matrixes.outputs.matrix_dockerimage_debug}}
 
-      buildspec_ubuntu:              ${{steps.matrixes.outputs.buildspec_ubuntu}}
-      buildspec_ubuntu_debug:        ${{steps.matrixes.outputs.buildspec_ubuntu_release}}
-      buildspec_ubuntu_release:      ${{steps.matrixes.outputs.buildspec_ubuntu_debug}}
-      buildspec_macos:               ${{steps.matrixes.outputs.buildspec_macos}}
-      buildspec_windows:             ${{steps.matrixes.outputs.buildspec_windows}}
-      buildspec_dockerimage_release: ${{steps.matrixes.outputs.buildspec_dockerimage_release}}
-      buildspec_dockerimage_debug:   ${{steps.matrixes.outputs.buildspec_dockerimage_debug}}
-
   ## Build docker image named 'hyperledger/iroha-builder' with all stuff to compile iroha and its dependancies
   ## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
   ## and conditional tags :edge for development branch, and :latest for git-tags.
@@ -318,18 +309,11 @@ jobs:
             type=ref,event=pr
             type=ref,event=tag
             type=schedule
-            type=edge,branch=develop
-          #  type=semver,pattern={{version}}
-          #  type=semver,pattern={{major}}.{{minor}}
-          #  type=sha,prefix=commit-,format=short
-          #  type=sha,prefix=commit-,format=long
         ## Docker image will be pushed with tags:
         ##  - hash of file Dockerfile.builder
         ##  - branchname, when branch is pushed
         ##  - pr-NUMBER, when pushed to PR
         ##  - git tag when tag is pushed
-        ##  - semver like 1.2.3 and 1.2 when tag vX.X.X is pushed
-        ##  - tag 'edge' when branch support/1.2.x is pushed
         ##  - schedule, see the docs
       - &step_docker_meta_ghcr
         <<: *step_docker_meta
@@ -380,29 +364,16 @@ jobs:
         name: Check if dockertaghash exists in remote registry
         id: dockertag_already
         run: |
-          ## FIXME page size is 100 and cannot be more, this needs to be extended in loop asking tags till the end
-          exists=$( curl -fL https://hub.docker.com/v2/repositories/$DOCKERHUB_ORG/iroha-builder/tags |
-            jq 'any( .results[]|.name == "${{env.dockertag}}" ; .)' )
-          echo "::set-output name=exists::$exists"
-
-          if test $exists = true ;then
-            tag=$dockertag
-          else
-            tag=edge
-          fi
-          container="$DOCKERHUB_ORG/iroha-builder:$tag"
-          echo "::set-output name=container::$container"
-          echo "::set-output name=container_tag::$tag"
+          echo "::set-output name=container::$DOCKERHUB_ORG/iroha-builder:$dockertag"
       -
-        if: ${{ steps.dockertag_already.outputs.container_tag != env.dockertag ||
-                steps.dockertag_already.outputs.container == '' }}
-        name: Possible ERROR, edited but not pushed
+        name: Possible ERROR, Dockerfile edited but image cannot be pushed
+        if: ${{ steps.docker_login.outcome != 'success' || steps.build_and_push.outcome != 'success' }}
         env:
           container: ${{steps.dockertag_already.outputs.container}}
           dockertag: ${{env.dockertag}}
         run: |
           cat <<END
-            ::error::Running on container '$container', YOUR CHANGES TO Dockerfile.builder WERE NOT APPLYED.
+            ::error::CHANGES TO Dockerfile.builder WERE NOT APPLYED.
 
             It seems container with tag '$dockertag' was not pushed to registry and does not exist remotely.
             The most possible reason is GitHub secrets are inaccessable to PRs from public forks.
@@ -425,12 +396,9 @@ jobs:
       ## WARN secret dropped from output!, output may not contain secret,
       ## and secret cannot be used in job:container directly, and there is no github non-secret variables...
       ## if dockertag is already pushed then use it. But let it be empty when tag does not exist remotely.
-      dockertag:   ${{steps.dockertag.outputs.dockertag}}
-      _dockertag: :${{steps.dockertag.outputs.dockertag}}
+      dockertag: ${{steps.dockertag.outputs.dockertag}}
+      container: ${{steps.dockertag_already.outputs.container}}
       pushed: ${{ steps.docker_login.outcome == 'success' && steps.build_and_push.outcome == 'success' }}
-      dockertag_already_exists: ${{steps.dockertag_already.outputs.exists}}
-      container:                ${{steps.dockertag_already.outputs.container}}
-      container_tag:            ${{steps.dockertag_already.outputs.container_tag}}
 
   ## Build iroha in a container made of the image earlier prepared
   ## Result artifacts are
@@ -440,9 +408,8 @@ jobs:
     needs:
       - Docker-iroha-builder
       - generate_matrixes
-    runs-on: [ self-hosted, Linux ] #ubuntu-latest
-    ## Container is taken from previous job
-    container: #&container
+    runs-on: [ self-hosted, Linux ]
+    container: ## Container is taken from previous job
       image: &container_image ${{needs.Docker-iroha-builder.outputs.container}}
       options: --user root
     strategy: &strategy_ubuntu_debug
@@ -461,36 +428,9 @@ jobs:
             ${{ toJson(needs) }}
           END
       -
-        run: test -n "$container"
-        env:
-          container: *container_image
-      -
-        if: ${{ needs.Docker-iroha-builder.outputs.container_tag != needs.Docker-iroha-builder.outputs.dockertag ||
-            needs.Docker-iroha-builder.outputs.container == '' }}
-        name: Possible WARNING
         env:
           container: *container_image
-          dockertag: ${{needs.Docker-iroha-builder.outputs.dockertag}}
-        run: |
-          cat <<END
-            ::warning::Running on container '$container', YOUR CHANGES TO Dockerfile.builder WERE NOT APPLYED.
-
-            It seems container with tag '$dockertag' was not pushed to registry and does not exist remotely.
-            The most possible reason is GitHub secrets are inaccessable to PRs from public forks.
-
-            $(test ${{github.event.pull_request.head.repo.full_name}} != ${{github.event.pull_request.base.repo.full_name}} \
-              && echo -n "SECRETS ARE NOT EXPOSED TO FORKS" || echo -n "SECRETS AVAILABLE")
-
-            **Consider to open PR from the same organization.**
-
-            What we know about this build:
-              - PR URL is ${{github.event.pull_request.html_url}}
-              - head repo is '${{github.event.pull_request.head.repo.full_name}}'
-              - base repo is '${{github.event.pull_request.base.repo.full_name}}'
-              - See more information in previous step 'Show context'
-
-            Please ask @kuvaldini on GitHub or in Telegram if you need help.
-          END
+        run: test -n "$container"
       - *step_system_info
       - *step_build_info
       - &step_env_from_buildspec
@@ -541,7 +481,6 @@ jobs:
       - &step_store_ccache_stats
         run: ccache --show-stats | tee /tmp/ccache-stats
       - &step_vcpkg_cache
-        #if: ${{false}}  ## This works bad when something patched or something updated, seems they does not recalc hash of changed packages. See todos in the begining of file.
         ## Read the docs https://vcpkg.readthedocs.io/en/latest/users/binarycaching/ https://github.com/microsoft/vcpkg/blob/master/docs/users/binarycaching.md
         name: Restore cache Vcpkg binarycache ## NOTE not useng NuGet because on ubuntu nuget needs mono of 433MB, unusable.
         uses: actions/cache@v2
@@ -810,7 +749,6 @@ jobs:
       - &step_brew_postgres
         name: Install Postgres on MacOS
         run: brew install postgresql
-        ## ToDo may be optimize, i.e. cache package
       - <<: *step_ctest
         timeout-minutes: 70
 
@@ -832,10 +770,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON( needs.generate_matrixes.outputs.matrix_windows ) }}
-    #if: ${{ false }}  ##FIXME Somehow cmake fails to find GTest and others
     if: ${{ false && ( fromJSON( needs.generate_matrixes.outputs.matrix_windows ).include[0] ) }}
-      # matrix:
-      #   BuildYype: [ Debug ] #,Release, RelWithDebInfo
     defaults:
       run:
         shell: bash  #pwsh
@@ -872,17 +807,16 @@ jobs:
       - *step_cmake_configure
       - *step_cmake_build
       - *step_always_after_build
-      -
-        name: Install Postgres on Windows
+      - name: Install Postgres on Windows
         run: choco install postgresql
       # - *step_ctest
 
   ## Build and publish docker image named 'hyperledger/iroha'  with irohad and iroha tools inside.
-  ## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
-  ## and conditional tags :edge (for development branches) and :latest (for git-tags)
+  ## The result docker image is pushed with tags :branch_name, :pr-NUMBER, :tag_name,
+  ## and :latest (for git-tags).
   ## Those docker image tags could be extended with suffixes with compiler and build type like
-  ## -gcc10, -clang, -debug, -gcc10-debug.
-  ## Result image name could look like: hyperledger/iroha:pr-1117, hyperledger/iroha-burrow:commit-XXXXX-debug
+  ## -gcc10, -clang, -debug, -gcc10-debug. Like :latest-debug, :main-debug, :1.3.0-gcc10-debug.
+  ## Result image name could look like: hyperledger/iroha:pr-1117, hyperledger/iroha:nightly.
   ## Note: image is push only when DockerHub login-token pair available - not to PRs from forks
   docker-R: &job_docker_image_release
     needs:
@@ -906,26 +840,28 @@ jobs:
       - *step_detect_commented_pr
       - *step_checkout
       - *step_artifact_suffix
-      - &step_download_artifact_iroha_deb
-        name: Download artifact
+      - name: Download artifact
         uses: actions/download-artifact@v2
         with:
           name: iroha-deb${{env.ARTIFACT_SUFFIX}}
-      - &rename_artifact_deb
-        name: Rename artifact debs
+      - name: Rename artifact debs
         run: |
           mv *iroha_shepherd.deb docker/release/iroha_shepherd.deb
           mv *irohad.deb         docker/release/iroha.deb
-      - &step_dockertag_release
-        <<: *step_docker_tag
+      - <<: *step_docker_tag
         env:
           dockertag: ${{ hashFiles('docker/release/**') }}
       - <<: *step_docker_meta
         with: &step_docker_release_meta_with
-          <<: *step_docker_meta_with
           images: |
             ${{ env.DOCKERHUB_ORG }}/${{ env.IMAGE_NAME }}${{ env._uses_suffix }}
-          flavor: suffix=${{env._compiler_suffix}}${{env._debug_suffix}}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=ref,event=tag
+            type=schedule
+          flavor: |
+            suffix=${{env._compiler_suffix}}${{env._debug_suffix}},onlatest=true
           #maybetodo flavor: prefix=${{ env.USES_PREFIX }}  ## In case creating repository hyperledger/iroha-burrow denied, Use tag prefix hyperledger/iroha:burrow-xxxx
       - <<: *step_docker_meta_ghcr
         with:

.github/chatops-gen-matrix.sh

@@ -68,7 +68,7 @@ handle_user_line(){
       return
    fi
    if [[ "${1:-}" != '/build' ]] ;then
-      echowarn "Line skipped, should start with '/build'"
+      echowarn "Line skipped, should start with '/build' : '${1:-}'"
       return
    fi
    shift
@@ -129,9 +129,9 @@ handle_user_line(){
             done
          done
       done
-      if test "$used_compilers" = ''; then
-         echowarn "No available compilers for '$os' among '$compilers', available: '$AVAILABLE_compilers'"
-      fi
+      # if test "$used_compilers" = ''; then
+      #    echowarn "No available compilers for '$os' among '$compilers', available: '$AVAILABLE_compilers'"
+      # fi
    done
 }
 
@@ -160,12 +160,6 @@ rm -f $ignored
 
 
 to_json(){
-   # echo "{
-   #       os:\"$1\",
-   #       cc:\"$2\",
-   #       BuildType:\"$3\",
-   #       CMAKE_USE:\"$( [[ "$4" = normal ]] || echo "-DUSE_${4^^}=ON" )\"
-   #    }"
    echo "{buildspec:\"$@\"}"
 }
 to_json_multiline(){
@@ -185,16 +179,6 @@ json_include(){
 MATRIX="$(echo "$MATRIX" | sed '/^$/d' | sort -uV)"
 echo "$MATRIX"
 
-echo "$MATRIX"                                                          >buildspec
-echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/'                         >buildspec_ubuntu
-echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/'            >buildspec_ubuntu_release
-echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /debug/'              >buildspec_ubuntu_debug
-echo "$MATRIX" | awk -v IGNORECASE=1 '/macos/'                          >buildspec_macos
-echo "$MATRIX" | awk -v IGNORECASE=1 '/windows/'                        >buildspec_windows
-## Build Docker images only with GCC-9 (mainstream compiler)
-echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/ && /gcc-9/' >buildspec_dockerimage_release
-echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /debug/   && /gcc-9/' >buildspec_dockerimage_debug
-
 echo "$MATRIX"                                                          | json_include >matrix
 echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/'                         | json_include >matrix_ubuntu
 echo "$MATRIX" | awk -v IGNORECASE=1 '/ubuntu/ && /release/'            | json_include >matrix_ubuntu_release