Skip to content

Run the CWL Workflow with calrissian

Goal

Run the app-water-body-cloud-native.1.0.0.cwl released application package using calrissian, a CWL runner for kubernetes.

calrissian creates a pod for each of the Workflow processing step.

Step 1 - Configure the workspace

The results produced will be available in the local folder /workspace/runs

terminal
1
2
3
4
export WORKSPACE=/workspace/app-package-training-bids23
export RUNTIME=/workspace/runs
mkdir -p ${RUNTIME}
cd ${RUNTIME}

Step 2 - Download the released Application package

version="1.0.0"

wget \
    -O /workspace/runs/app-water-bodies-cloud-native.${version}.cwl \
    https://github.com/Terradue/app-package-training-bids23/releases/download/${version}/app-water-bodies-cloud-native.${version}.cwl

sh ${WORKSPACE}/scripts/download-app-water-bodies-cloud-native.sh

Step 3 - Execute the Application Package

mkdir -p /calrissian/logs

version="1.0.0"

calrissian \
    --stdout /calrissian/results.json \
    --stderr /calrissian/app.log \
    --max-ram 4G \
    --max-cores "8" \
    --tmp-outdir-prefix /calrissian/tmp \
    --outdir /calrissian/results \
    --usage-report /calrissian/usage.json \
    --tool-logs-basepath /calrissian/logs \
    --pod-nodeselectors /etc/calrissian/pod-node-selector.yaml \
    /workspace/runs/app-water-bodies-cloud-native.${version}.cwl \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210708_0_L2A" \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_10TFK_20210713_0_L2A" \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20210718_0_L2A" \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220524_0_L2A" \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220514_0_L2A" \
    --stac_items "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2A_10TFK_20220504_0_L2A" \
    --aoi="-121.399,39.834,-120.74,40.472" \
    --epsg "EPSG:4326"

sh ${WORKSPACE}/scripts/calrissian-cloud-native.sh

New pods are created:

(base) jovyan@coder-mrossi:~/app-package-training-bids23$ kubectl get pods
NAME                       READY   STATUS              RESTARTS   AGE
coder-mrossi               1/1     Running             0          1h
node-crop-2-pod-laqxbhhy   0/1     ContainerCreating   0          8s
node-crop-3-pod-gvdnujzn   0/1     ContainerCreating   0          8s
node-crop-4-pod-qisbfayc   0/1     ContainerCreating   0          8s
node-crop-5-pod-joficzgy   0/1     ContainerCreating   0          8s
node-crop-6-pod-ejarwpxk   0/1     ContainerCreating   0          8s
node-crop-7-pod-ecjgjglq   0/1     ContainerCreating   0          8s
node-crop-pod-ebgkkvpc     0/1     ContainerCreating   0          8s

Step 4 - Inspect the results

tree $( cat /calrissian/results.json | jq -r .stac_catalog.path )

Expected outcome

(base) jovyan@coder-mrossi:~/runs$ tree $( cat /calrissian/results.json | jq -r .stac_catalog.path )
/calrissian/results/tmp3vhr5k8r
├── S2A_10TFK_20210708_0_L2A
│   ├── S2A_10TFK_20210708_0_L2A.json
│   └── otsu.tif
├── S2A_10TFK_20210718_0_L2A
│   ├── S2A_10TFK_20210718_0_L2A.json
│   └── otsu.tif
├── S2A_10TFK_20220504_0_L2A
│   ├── S2A_10TFK_20220504_0_L2A.json
│   └── otsu.tif
├── S2A_10TFK_20220514_0_L2A
│   ├── S2A_10TFK_20220514_0_L2A.json
│   └── otsu.tif
├── S2A_10TFK_20220524_0_L2A
│   ├── S2A_10TFK_20220524_0_L2A.json
│   └── otsu.tif
├── S2B_10TFK_20210713_0_L2A
│   ├── S2B_10TFK_20210713_0_L2A.json
│   └── otsu.tif
└── catalog.json

6 directories, 13 files