Commit cc806b74 authored by Vignesh Raman's avatar Vignesh Raman Committed by Helen Koike
Browse files

drm/ci: uprev mesa version 

zlib.net is not allowing tarball download anymore and results
in below error in kernel+rootfs_arm32 container build,
urllib.error.HTTPError: HTTP Error 403: Forbidden
urllib.error.HTTPError: HTTP Error 415: Unsupported Media Type

Uprev mesa to latest version which includes a fix for this issue.
https://gitlab.freedesktop.org/mesa/mesa/-/commit/908f444e



Use id_tokens for JWT authentication. Since s3 bucket is migrated to
mesa-rootfs, update the variables accordingly. Also copy helper scripts
to install, so that the ci jobs can use these scripts for logging.

Signed-off-by: default avatarVignesh Raman <vignesh.raman@collabora.com>
Reviewed-by: default avatarDmitry Baryshkov <dmitry.baryshkov@linaro.org>
Reviewed-by: default avatarJessica Zhang <quic_jesszhan@quicinc.com>
Acked-by: default avatarHelen Koike <helen.koike@collabora.com>
Signed-off-by: default avatarHelen Koike <helen.koike@collabora.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20240529024049.356327-2-vignesh.raman@collabora.com
parent 3ed96977

drivers/gpu/drm/ci/build-igt.sh

+1 −1
Original line number Diff line number Diff line
@@ -32,4 +32,4 @@ tar -cf artifacts/igt.tar /igt 
# Pass needed files to the test stage

S3_ARTIFACT_NAME="igt.tar.gz"

gzip -c artifacts/igt.tar > ${S3_ARTIFACT_NAME}

ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${KERNEL_ARCH}/${S3_ARTIFACT_NAME}
 
ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${KERNEL_ARCH}/${S3_ARTIFACT_NAME}

drivers/gpu/drm/ci/build.sh

+4 −2
Original line number Diff line number Diff line
@@ -131,6 +131,7 @@ fi 
# Pass needed files to the test stage

mkdir -p install

cp -rfv .gitlab-ci/* install/.

cp -rfv ci/*  install/.

cp -rfv install/common install/ci-common

cp -rfv drivers/gpu/drm/ci/* install/.
@@ -144,14 +145,15 @@ if [[ "$UPLOAD_TO_MINIO" = "1" ]]; then 
        FILES_TO_UPLOAD="$FILES_TO_UPLOAD $(basename -a $DEVICE_TREES)"

    fi



    ls -l "${S3_JWT_FILE}"

    for f in $FILES_TO_UPLOAD; do

        ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" /lava-files/$f \

        ci-fairy s3cp --token-file "${S3_JWT_FILE}" /lava-files/$f \

                https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/$f

    done



    S3_ARTIFACT_NAME="kernel-files.tar.zst"

    tar --zstd -cf $S3_ARTIFACT_NAME install

    ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/${S3_ARTIFACT_NAME}

    ci-fairy s3cp --token-file "${S3_JWT_FILE}" ${S3_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/${S3_ARTIFACT_NAME}



    echo "Download vmlinux.xz from https://${PIPELINE_ARTIFACTS_BASE}/${DEBIAN_ARCH}/vmlinux.xz"

fi

drivers/gpu/drm/ci/container.yml

+4 −8
Original line number Diff line number Diff line
@@ -36,15 +36,15 @@ debian/android_build: 
  rules:

    - when: never



debian/x86_64_test-android:

.debian/x86_64_test-android:

  rules:

    - when: never



windows_build_vs2019:

windows_build_msvc:

  rules:

    - when: never



windows_test_vs2019:

windows_test_msvc:

  rules:

    - when: never
@@ -56,10 +56,6 @@ rustfmt: 
   rules:

    - when: never



windows_vs2019:

   rules:

    - when: never



clang-format:

windows_msvc:

   rules:

    - when: never
 
 No newline at end of file

drivers/gpu/drm/ci/gitlab-ci.yml

+30 −14
Original line number Diff line number Diff line 
variables:

  DRM_CI_PROJECT_PATH: &drm-ci-project-path mesa/mesa

  DRM_CI_COMMIT_SHA: &drm-ci-commit-sha 9d162de9a05155e1c4041857a5848842749164cf

  DRM_CI_COMMIT_SHA: &drm-ci-commit-sha e2b9c5a9e3e4f9b532067af8022eaef8d6fc6c00



  UPSTREAM_REPO: git://anongit.freedesktop.org/drm/drm

  TARGET_BRANCH: drm-next
@@ -19,33 +19,47 @@ variables: 
          bash download-git-cache.sh

          rm download-git-cache.sh

          set +o xtrace

  S3_JWT_FILE: /s3_jwt

  S3_HOST: s3.freedesktop.org

  # This bucket is used to fetch the kernel image

  S3_KERNEL_BUCKET: mesa-rootfs

  # Bucket for git cache

  S3_GITCACHE_BUCKET: git-cache

  # Bucket for the pipeline artifacts pushed to S3

  S3_ARTIFACTS_BUCKET: artifacts

  # per-pipeline artifact storage on MinIO

  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}

  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}

  # per-job artifact storage on MinIO

  JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}

  # default kernel for rootfs before injecting the current kernel tree

  KERNEL_REPO: "gfx-ci/linux"

  KERNEL_TAG: "v6.6.4-for-mesa-ci-e4f4c500f7fb"

  KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}

  KERNEL_TAG: "v6.6.21-mesa-f8ea"

  KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}

  PKG_REPO_REV: "3cc12a2a"

  LAVA_TAGS: subset-1-gfx

  LAVA_JOB_PRIORITY: 30

  ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts

  # Python scripts for structured logger

  PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"



default:

  id_tokens:

    S3_JWT:

      aud: https://s3.freedesktop.org

  before_script:

    - export SCRIPTS_DIR=$(mktemp -d)

    - curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${DRM_CI_PROJECT_URL}/-/raw/${DRM_CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh"

    - source ${SCRIPTS_DIR}/setup-test-env.sh

    - echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"

    - export CI_JOB_JWT_FILE="${CI_JOB_JWT_FILE:-$(mktemp)}"

    - echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"

    - unset CI_JOB_JWT

    - echo -n "${S3_JWT}" > "${S3_JWT_FILE}"

    - unset CI_JOB_JWT S3_JWT

    - echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"



    - echo -e "\e[0Ksection_start:$(date +%s):drm_ci_download_section[collapsed=true]\r\e[0KDownloading mesa from $DRM_CI_PROJECT_URL/-/archive/$DRM_CI_COMMIT_SHA/mesa-$DRM_CI_COMMIT_SHA.tar.gz"

    - cd $CI_PROJECT_DIR

    - curl --output - $DRM_CI_PROJECT_URL/-/archive/$DRM_CI_COMMIT_SHA/mesa-$DRM_CI_COMMIT_SHA.tar.gz | tar -xz

    - mv mesa-$DRM_CI_COMMIT_SHA/.gitlab-ci* .

    - mv mesa-$DRM_CI_COMMIT_SHA/bin/ci .

    - rm -rf mesa-$DRM_CI_COMMIT_SHA/

    - echo -e "\e[0Ksection_end:$(date +%s):drm_ci_download_section\r\e[0K"
@@ -53,9 +67,9 @@ default: 
    - >

      set +x



      test -e "${CI_JOB_JWT_FILE}" &&

      export CI_JOB_JWT="$(<${CI_JOB_JWT_FILE})" &&

      rm "${CI_JOB_JWT_FILE}"

      test -e "${S3_JWT_FILE}" &&

      export S3_JWT="$(<${S3_JWT_FILE})" &&

      rm "${S3_JWT_FILE}"



include:

  - project: 'freedesktop/ci-templates'
@@ -87,6 +101,7 @@ include: 
      - '/src/intel/ci/gitlab-ci-inc.yml'

      - '/src/freedreno/ci/gitlab-ci-inc.yml'

      - '/src/amd/ci/gitlab-ci-inc.yml'

      - '/src/virtio/ci/gitlab-ci-inc.yml'

  - drivers/gpu/drm/ci/image-tags.yml

  - drivers/gpu/drm/ci/container.yml

  - drivers/gpu/drm/ci/static-checks.yml
@@ -98,6 +113,7 @@ include: 
stages:

  - sanity

  - container

  - code-validation

  - git-archive

  - build

  - amdgpu
@@ -107,7 +123,6 @@ stages: 
  - msm

  - rockchip

  - virtio-gpu

  - lint



# YAML anchors for rule conditions

# --------------------------------
@@ -218,14 +233,15 @@ make git archive: 
  script:

    # Remove drm-ci files we just added

    - rm -rf .gitlab-ci.*

    - rm -rf ci



    # Compactify the .git directory

    - git gc --aggressive

    # compress the current folder

    - tar -cvzf ../$CI_PROJECT_NAME.tar.gz .



    # login with the JWT token file

    - ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz

    # Use id_tokens for JWT auth

    - ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/${S3_GITCACHE_BUCKET}/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz





# Sanity checks of MR settings and commit logs

drivers/gpu/drm/ci/image-tags.yml

+1 −1
Original line number Diff line number Diff line 
variables:

   CONTAINER_TAG: "2023-10-11-mesa-uprev"

   CONTAINER_TAG: "2024-05-09-mesa-uprev"

   DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base"

   DEBIAN_BASE_TAG: "${CONTAINER_TAG}"