# Insula Workflow Client
## Example
```python
from InsulaWorkflowClient import InsulaOpenIDConnect
from InsulaWorkflowClient.InsulaApiConfig import InsulaApiConfig
from InsulaWorkflowClient.InsulaClient import InsulaClient
insula_auth: InsulaOpenIDConnect = InsulaOpenIDConnect(
authorization_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/auth',
token_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/token',
redirect_uri='http://localhost:9207/auth',
client_id="api-client"
)
insula_auth.set_user_credentials(username="<USER>", password="<PASSWORD>")
ic = InsulaClient(InsulaApiConfig(
'https://iride-lot3.ope.insula.earth',
insula_auth,
status_polling_interval=30, # Seconds
interval_between_requests=5, # Seconds
max_processor_attempts=3
))
ic.run_from_file('<THE TEMPLATE PATH>',
{
'access_key': '<access_key>',
'secret_key': '<secret_key>',
'elaboration_date': '2023_2024',
'region_name': 'Molise'
}, '<optional log name>')
```
```yaml
type: workflow
language: native
name: work_iride
version: beta/1
description: |
PLEASE, DO NOT CHANGE THIS FILE
pre processing workflow from step 0a to Siam, push SPOT_T0, SPOT_T1
portal_ref: eopaas://refData/33/workflow_pre_processing.yaml
parameters:
#----dynamic----
id_tile: '552'
elaboration_date: '2023_2024'
time: 'SPOT_T1'
#----static----
reticle_definition: 'eopaas://refData/33/reticolo_20x20km_ita_epsg4326.geojson'
collection_base: '{"out":"iridelot33c67fd6450604bfebfa541474a094a62"}'
siam_collection_base: '{"calibration_data":"iridelot33c67fd6450604bfebfa541474a094a62","masks":"iridelot33c67fd6450604bfebfa541474a094a62","pct_full":"iridelot33c67fd6450604bfebfa541474a094a62","pct_33":"iridelot33c67fd6450604bfebfa541474a094a62"}'
s3_platform_ref: 'eopaas://refData/33/obs.eu-de.otc.t-systems.com_bot.json'
#for s3copy processor without https://
s3_copy_endpoint: 'hhf.eu-dfd.otc.t-fgfd.com'
s3_copy_bucket_map_sat: 'hhs.fs-de.ffd.t-systems.com'
s3_bucket_map_cgi: 'csa-fdf-svc-hh-fg-soil-sealing-map-cgi'
configuration:
max_parallel_jobs: 5
continue_on_error: false
delete_workflow_log: false
requirements:
connections:
- name: s3_soil_sealing
type: s3
params:
access_key: ${workflow.param.access_key}
secret_key: ${workflow.param.secret_key}
endpoint: 'https://sfd.ff-fgd.otc.t-dgfd.com'
bucket: '${workflow.param.s3_bucket_map_cgi}'
steps:
- - name: create_buckets_name
type: script
continue_on_error: false
outputs:
- s4_03_t0_siam
- s4_02_t1_siam
- soil_s4_03_First_processing
- soil_s4_02_First_processing
- pid_path
params:
- name: id_tile
values: ${workflow.param.id_tile}
- name: elaboration_date
values: ${workflow.param.elaboration_date}
- name: time
values: ${workflow.param.time}
source: |
s4_03_t0_siam = f'{elaboration_date}/S4_03/{id_tile}/Pre_Processing/siam'
s4_02_t1_siam = f'{elaboration_date}/S4_02/{id_tile}/Pre_Processing/siam'
soil_s4_03_First_processing = f'{elaboration_date}/S4_03/{id_tile}/First_Processing'
soil_s4_02_First_processing = f'{elaboration_date}/S4_02/{id_tile}/First_Processing'
pid_path = f'{elaboration_date}/__processing/{id_tile}/First_Processing'
print(f'Pre processing s4_03 siam s3 path: {s4_03_t0_siam}')
print(f'Pre processing s4_02 siam s3 path: {s4_02_t0_siam}')
print(f'soil_s4_03_First_processing s3 path: {soil_s4_03_First_processing}')
print(f'soil_s4_02_First_processing s3 path: {soil_s4_02_First_processing}')
- - name: IrideLot3ClippingReticle
type: processor
service_id: '8'
params:
- name: id_tile
values: '${workflow.param.idtile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: s3_platform_ref
values: '${workflow.param.s3_platform_ref}'
- name: reticle_definition
values: '${workflow.param.reticle_definition}'
- name: collection
values: '${workflow.param.collection_base}'
- - name: IrideLot3Clipping
type: processor
service_id: '9'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: SPOT_T0
values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T0}'
- name: SPOT_T1
values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T1}'
- name: collection
values: '${workflow.param.collection_base}'
- - name: siam_spot_t0
template: siam
params:
- name: parallelInputs
values: '${workflow.step.IrideLot3Clipping.SPOT_T0}'
- name: siam_spot_t1
template: siam
params:
- name: parallelInputs
values: '${workflow.step.IrideLot3Clipping.SPOT_T1}'
- name: IrideLot3SoilSealing
type: processor
service_id: '10'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: reticolo
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
- name: siam
values: ${workflow.step.siam_spot_t0} $[(.*61SpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: mapsat
values: 'none'
- name: collection
values: '${workflow.param.collection_base}'
- name: IrideLot3ChangeDetectionMergeSiam
type: processor
service_id: '12'
params:
- name: id_tile
values: '${workflow.param.id_tile}'
- name: elaboration_date
values: '${workflow.param.elaboration_date}'
- name: reticolo_t0
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}
- name: reticolo_t1
values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T1}
- name: siam_t0
values: ${workflow.step.siam_spot_t0} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: siam_t1
values: ${workflow.step.siam_spot_t1} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\.tif$)|(.*VegBinaryMask.*\.dat$)|(.*VegBinaryMask.*\.hdr$)]
- name: collection
values: '${workflow.param.collection_base}'
- - name: SiamChangeDetector
type: processor
service_id: '14'
params:
- name: T1
values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t0} $[(.*\.dat$)|(.*\.hdr$)]
- name: T2
values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t1} $[(.*\.dat$)|(.*\.hdr$)]
- name: collection
values: '${workflow.param.collection_base}'
- - name: siam_push_t0
template: s3_copy
params:
- name: source
values: '${workflow.step.siam_spot_t0}'
- name: destination
values: '${workflow.step.create_buckets_name.s4_03_t0_siam}'
- name: siam_push_t1
template: s3_copy
params:
- name: source
values: '${workflow.step.siam_spot_t1}'
- name: destination
values: '${workflow.step.create_buckets_name.s4_02_t1_siam}'
- name: IrideLot3SoilSealing_push_t0
template: s3_copy
params:
- name: source
values: '${workflow.step.IrideLot3SoilSealing}'
- name: destination
values: '${workflow.step.create_buckets_name.soil_s4_03_First_processing}'
- name: SiamChangeDetector_push_t1
template: s3_copy
params:
- name: source
values: '${workflow.step.SiamChangeDetector}'
- name: destination
values: '${workflow.step.create_buckets_name.soil_s4_02_First_processing}'
- - name: create_pids_file
type: script
continue_on_error: true
outputs:
- file_pid
params:
- name: job_reticle_id
values: ${status.step.IrideLot3ClippingReticle.job_id}
- name: job_siam_spot_t0_id
values: ${status.step.siam_spot_t0.job_id}
- name: job_siam_spot_t1_id
values: ${status.step.siam_spot_t1.job_id}
source: |
content = f'{{"job_reticle_id":"{job_reticle_id}", "job_siam_spot_t0_id":"{job_siam_spot_t0_id}", "job_siam_spot_t1_id":"{job_siam_spot_t1_id}" }}'
print(f'{content}')
with open('jobs_file.json', 'w') as jobs_file:
jobs_file.write(content)
print(f'{content}')
file_pid = 'jobs_file.json'
- - name: push_ids
type: s3
connection: s3_soil_sealing
continue_on_error: false
action: push
params:
- name: in
values: '${workflow.step.create_pids_file.file_pid}'
save_in: '2023_2024/S4-03/TEST'
templates:
- name: s3_copy
service_id: '11'
type: processor
params:
- name: action
values: upload
- name: s3_endpoint
values: '${workflow.param.s3_copy_endpoint}'
- name: s3_bucket
values: '${workflow.param.s3_bucket_map_cgi}'
- name: s3_platform_ref
values: '${workflow.param.s3_platform_ref}'
- name: collection
values: '{"output":"iridelot365b7d9dde6044c00b26abe046aed0556"}'
- name: siam
service_id: '3'
type: processor
params:
- name: veg_mask
values: 'binary'
- name: cloud_mask
values: 'binary'
- name: urban_mask
values: 'disabled'
- name: collection
values: '${workflow.param.siam_collection_base}'
- name: shadow_mask
values: 'enabled'
- name: keep_calibration_data
values: 'false'
- name: extra_indexes
values: 'true'
- name: water_mask
values: 'disabled'
- name: bare_soil_mask
values: 'disabled'
- name: burned_area_mask
values: 'disabled'
- name: use_no_data_mask
values: 'true'
- name: smoke_mask
values: 'disabled'
- name: classification_mode
values: 'crisp'
```
Raw data
{
"_id": null,
"home_page": null,
"name": "InsulaWorkflowClient",
"maintainer": null,
"docs_url": null,
"requires_python": ">=3.9",
"maintainer_email": "Roberto Di Rienzo <roberto.dirienzo@cgi.com>",
"keywords": "Italia, cgi, client, insula",
"author": null,
"author_email": "Roberto Di Rienzo <roberto.dirienzo@codelithic.com>",
"download_url": null,
"platform": null,
"description": "# Insula Workflow Client\n\n## Example\n\n```python\nfrom InsulaWorkflowClient import InsulaOpenIDConnect\nfrom InsulaWorkflowClient.InsulaApiConfig import InsulaApiConfig\nfrom InsulaWorkflowClient.InsulaClient import InsulaClient\n\ninsula_auth: InsulaOpenIDConnect = InsulaOpenIDConnect(\n authorization_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/auth',\n token_endpoint='https://identity.ope.insula.earth//realms/iride-lot3/protocol/openid-connect/token',\n redirect_uri='http://localhost:9207/auth',\n client_id=\"api-client\"\n)\ninsula_auth.set_user_credentials(username=\"<USER>\", password=\"<PASSWORD>\")\n\nic = InsulaClient(InsulaApiConfig(\n 'https://iride-lot3.ope.insula.earth',\n insula_auth,\n status_polling_interval=30, # Seconds\n interval_between_requests=5, # Seconds\n max_processor_attempts=3\n))\n\nic.run_from_file('<THE TEMPLATE PATH>',\n {\n 'access_key': '<access_key>',\n 'secret_key': '<secret_key>',\n 'elaboration_date': '2023_2024',\n 'region_name': 'Molise'\n }, '<optional log name>')\n```\n\n```yaml\ntype: workflow\nlanguage: native\nname: work_iride\nversion: beta/1\ndescription: |\n PLEASE, DO NOT CHANGE THIS FILE \n pre processing workflow from step 0a to Siam, push SPOT_T0, SPOT_T1\n\nportal_ref: eopaas://refData/33/workflow_pre_processing.yaml\n\nparameters:\n #----dynamic----\n id_tile: '552'\n elaboration_date: '2023_2024'\n time: 'SPOT_T1'\n #----static----\n reticle_definition: 'eopaas://refData/33/reticolo_20x20km_ita_epsg4326.geojson'\n collection_base: '{\"out\":\"iridelot33c67fd6450604bfebfa541474a094a62\"}'\n siam_collection_base: '{\"calibration_data\":\"iridelot33c67fd6450604bfebfa541474a094a62\",\"masks\":\"iridelot33c67fd6450604bfebfa541474a094a62\",\"pct_full\":\"iridelot33c67fd6450604bfebfa541474a094a62\",\"pct_33\":\"iridelot33c67fd6450604bfebfa541474a094a62\"}'\n s3_platform_ref: 'eopaas://refData/33/obs.eu-de.otc.t-systems.com_bot.json'\n #for s3copy processor without https://\n s3_copy_endpoint: 'hhf.eu-dfd.otc.t-fgfd.com'\n s3_copy_bucket_map_sat: 'hhs.fs-de.ffd.t-systems.com'\n s3_bucket_map_cgi: 'csa-fdf-svc-hh-fg-soil-sealing-map-cgi'\n\nconfiguration:\n max_parallel_jobs: 5\n continue_on_error: false\n delete_workflow_log: false\n\nrequirements:\n connections:\n - name: s3_soil_sealing\n type: s3\n params:\n access_key: ${workflow.param.access_key}\n secret_key: ${workflow.param.secret_key}\n endpoint: 'https://sfd.ff-fgd.otc.t-dgfd.com'\n bucket: '${workflow.param.s3_bucket_map_cgi}'\n\nsteps:\n - - name: create_buckets_name\n type: script\n continue_on_error: false\n outputs:\n - s4_03_t0_siam\n - s4_02_t1_siam\n - soil_s4_03_First_processing\n - soil_s4_02_First_processing\n - pid_path\n params:\n - name: id_tile\n values: ${workflow.param.id_tile}\n - name: elaboration_date\n values: ${workflow.param.elaboration_date}\n - name: time\n values: ${workflow.param.time}\n source: |\n\n s4_03_t0_siam = f'{elaboration_date}/S4_03/{id_tile}/Pre_Processing/siam'\n s4_02_t1_siam = f'{elaboration_date}/S4_02/{id_tile}/Pre_Processing/siam' \n soil_s4_03_First_processing = f'{elaboration_date}/S4_03/{id_tile}/First_Processing'\n soil_s4_02_First_processing = f'{elaboration_date}/S4_02/{id_tile}/First_Processing'\n pid_path = f'{elaboration_date}/__processing/{id_tile}/First_Processing'\n\n print(f'Pre processing s4_03 siam s3 path: {s4_03_t0_siam}') \n print(f'Pre processing s4_02 siam s3 path: {s4_02_t0_siam}') \n print(f'soil_s4_03_First_processing s3 path: {soil_s4_03_First_processing}')\n print(f'soil_s4_02_First_processing s3 path: {soil_s4_02_First_processing}')\n\n\n - - name: IrideLot3ClippingReticle\n type: processor\n service_id: '8'\n params:\n - name: id_tile\n values: '${workflow.param.idtile}'\n - name: elaboration_date\n values: '${workflow.param.elaboration_date}'\n - name: s3_platform_ref\n values: '${workflow.param.s3_platform_ref}'\n - name: reticle_definition\n values: '${workflow.param.reticle_definition}'\n - name: collection\n values: '${workflow.param.collection_base}'\n\n - - name: IrideLot3Clipping\n type: processor\n service_id: '9'\n params:\n - name: id_tile\n values: '${workflow.param.id_tile}'\n - name: elaboration_date\n values: '${workflow.param.elaboration_date}'\n - name: SPOT_T0\n values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T0}'\n - name: SPOT_T1\n values: '${workflow.step.IrideLot3ClippingReticle.SPOT_T1}'\n - name: collection\n values: '${workflow.param.collection_base}'\n\n - - name: siam_spot_t0\n template: siam\n params:\n - name: parallelInputs\n values: '${workflow.step.IrideLot3Clipping.SPOT_T0}'\n - name: siam_spot_t1\n template: siam\n params:\n - name: parallelInputs\n values: '${workflow.step.IrideLot3Clipping.SPOT_T1}'\n\n - name: IrideLot3SoilSealing\n type: processor\n service_id: '10'\n params:\n - name: id_tile\n values: '${workflow.param.id_tile}'\n - name: elaboration_date\n values: '${workflow.param.elaboration_date}'\n - name: reticolo\n values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}\n - name: siam\n values: ${workflow.step.siam_spot_t0} $[(.*61SpCt.*\\.tif$)|(.*VegBinaryMask.*\\.dat$)|(.*VegBinaryMask.*\\.hdr$)]\n - name: mapsat\n values: 'none'\n - name: collection\n values: '${workflow.param.collection_base}'\n\n - name: IrideLot3ChangeDetectionMergeSiam\n type: processor\n service_id: '12'\n params:\n - name: id_tile\n values: '${workflow.param.id_tile}'\n - name: elaboration_date\n values: '${workflow.param.elaboration_date}'\n - name: reticolo_t0\n values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T0}\n - name: reticolo_t1\n values: ${workflow.step.IrideLot3ClippingReticle.SPOT_T1}\n - name: siam_t0\n values: ${workflow.step.siam_spot_t0} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\\.tif$)|(.*VegBinaryMask.*\\.dat$)|(.*VegBinaryMask.*\\.hdr$)]\n - name: siam_t1\n values: ${workflow.step.siam_spot_t1} $[(.*fRatioGreennessIndex.*)|(.*fRatioCanopyChlorophyllContent.*)|(.*33SharedSpCt.*\\.tif$)|(.*VegBinaryMask.*\\.dat$)|(.*VegBinaryMask.*\\.hdr$)]\n - name: collection\n values: '${workflow.param.collection_base}'\n\n - - name: SiamChangeDetector\n type: processor\n service_id: '14'\n params:\n - name: T1\n values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t0} $[(.*\\.dat$)|(.*\\.hdr$)]\n - name: T2\n values: ${workflow.step.IrideLot3ChangeDetectionMergeSiam.merge_t1} $[(.*\\.dat$)|(.*\\.hdr$)]\n - name: collection\n values: '${workflow.param.collection_base}'\n\n\n - - name: siam_push_t0\n template: s3_copy\n params:\n - name: source\n values: '${workflow.step.siam_spot_t0}'\n - name: destination\n values: '${workflow.step.create_buckets_name.s4_03_t0_siam}'\n\n - name: siam_push_t1\n template: s3_copy\n params:\n - name: source\n values: '${workflow.step.siam_spot_t1}'\n - name: destination\n values: '${workflow.step.create_buckets_name.s4_02_t1_siam}'\n\n - name: IrideLot3SoilSealing_push_t0\n template: s3_copy\n params:\n - name: source\n values: '${workflow.step.IrideLot3SoilSealing}'\n - name: destination\n values: '${workflow.step.create_buckets_name.soil_s4_03_First_processing}'\n\n - name: SiamChangeDetector_push_t1\n template: s3_copy\n params:\n - name: source\n values: '${workflow.step.SiamChangeDetector}'\n - name: destination\n values: '${workflow.step.create_buckets_name.soil_s4_02_First_processing}'\n\n\n - - name: create_pids_file\n type: script\n continue_on_error: true\n outputs:\n - file_pid\n params:\n - name: job_reticle_id\n values: ${status.step.IrideLot3ClippingReticle.job_id}\n - name: job_siam_spot_t0_id\n values: ${status.step.siam_spot_t0.job_id}\n - name: job_siam_spot_t1_id\n values: ${status.step.siam_spot_t1.job_id}\n source: |\n\n content = f'{{\"job_reticle_id\":\"{job_reticle_id}\", \"job_siam_spot_t0_id\":\"{job_siam_spot_t0_id}\", \"job_siam_spot_t1_id\":\"{job_siam_spot_t1_id}\" }}'\n print(f'{content}')\n with open('jobs_file.json', 'w') as jobs_file:\n jobs_file.write(content) \n print(f'{content}')\n file_pid = 'jobs_file.json'\n\n\n - - name: push_ids\n type: s3\n connection: s3_soil_sealing\n continue_on_error: false\n action: push\n params:\n - name: in\n values: '${workflow.step.create_pids_file.file_pid}'\n save_in: '2023_2024/S4-03/TEST'\n\ntemplates:\n\n - name: s3_copy\n service_id: '11'\n type: processor\n params:\n - name: action\n values: upload\n - name: s3_endpoint\n values: '${workflow.param.s3_copy_endpoint}'\n - name: s3_bucket\n values: '${workflow.param.s3_bucket_map_cgi}'\n - name: s3_platform_ref\n values: '${workflow.param.s3_platform_ref}'\n - name: collection\n values: '{\"output\":\"iridelot365b7d9dde6044c00b26abe046aed0556\"}'\n\n\n - name: siam\n service_id: '3'\n type: processor\n params:\n - name: veg_mask\n values: 'binary'\n - name: cloud_mask\n values: 'binary'\n - name: urban_mask\n values: 'disabled'\n - name: collection\n values: '${workflow.param.siam_collection_base}'\n - name: shadow_mask\n values: 'enabled'\n - name: keep_calibration_data\n values: 'false'\n - name: extra_indexes\n values: 'true'\n - name: water_mask\n values: 'disabled'\n - name: bare_soil_mask\n values: 'disabled'\n - name: burned_area_mask\n values: 'disabled'\n - name: use_no_data_mask\n values: 'true'\n - name: smoke_mask\n values: 'disabled'\n - name: classification_mode\n values: 'crisp'\n\n```",
"bugtrack_url": null,
"license": null,
"summary": "Insula Workflow CLient",
"version": "0.7.7",
"project_urls": null,
"split_keywords": [
"italia",
" cgi",
" client",
" insula"
],
"urls": [
{
"comment_text": null,
"digests": {
"blake2b_256": "91fe0bfa9286e7ff18fc5c7a3d8f3e11aa6025c65a30a9b82badc8e5dcd8fe6f",
"md5": "c26e578fb965dcfb7c235d52034fe31f",
"sha256": "cb292f859c3af9a0e12f2ebd39045e86e3bc75aa773aa5f054f028c1a55dd398"
},
"downloads": -1,
"filename": "insulaworkflowclient-0.7.7-py3-none-any.whl",
"has_sig": false,
"md5_digest": "c26e578fb965dcfb7c235d52034fe31f",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.9",
"size": 26766,
"upload_time": "2025-02-21T08:13:20",
"upload_time_iso_8601": "2025-02-21T08:13:20.503228Z",
"url": "https://files.pythonhosted.org/packages/91/fe/0bfa9286e7ff18fc5c7a3d8f3e11aa6025c65a30a9b82badc8e5dcd8fe6f/insulaworkflowclient-0.7.7-py3-none-any.whl",
"yanked": false,
"yanked_reason": null
}
],
"upload_time": "2025-02-21 08:13:20",
"github": false,
"gitlab": false,
"bitbucket": false,
"codeberg": false,
"lcname": "insulaworkflowclient"
}
JSON