theisuru-patch-15

No remote found

Workflow

The following workflow defines all tasks necessary to upload, preprocess, share, and map a new data dictionary.

Upload cohort data
Open Google Sheet
Run automated mapping for new data dictionary
Share Google Sheet with submitter
Prepare data dictionary for build
Run automated validation
Build data dictionary
View results
Add data dictionary to Version Control
Prepare git commit (click on Commit in Version menu)
Push changes to GitHub (click on Push in Version menu), and make pull request.
Delete Google sheet (Caution, cannot be undone)

IHCC Data Admin Tasks

Console

Action all started at 2022-12-01T12:36:06.770Z (2022-12-01T12:36:06.770Z)

Success

$ make -f Makefile all

python3 src/create_xref_template.py build/chop.tsv build/intermediate/gecko_index.tsv build/intermediate/chop-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/chop.tsv \
template \
--template build/intermediate/chop-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/chop.owl" \
--version-iri "https://purl.ihccglobal.org/chop/releases/2022-12-01/chop.owl" \
--annotation-file metadata/chop.ttl \
--output build/chop.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/ge.tsv | tr '#' '\n' > build/ge.tsv
python3 src/create_xref_template.py build/ge.tsv build/intermediate/gecko_index.tsv build/intermediate/ge-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/ge.tsv \
template \
--template build/intermediate/ge-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/ge.owl" \
--version-iri "https://purl.ihccglobal.org/ge/releases/2022-12-01/ge.owl" \
--annotation-file metadata/ge.ttl \
--output build/ge.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/uklwc.tsv | tr '#' '\n' > build/uklwc.tsv
python3 src/create_xref_template.py build/uklwc.tsv build/intermediate/gecko_index.tsv build/intermediate/uklwc-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/uklwc.tsv \
template \
--template build/intermediate/uklwc-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/uklwc.owl" \
--version-iri "https://purl.ihccglobal.org/uklwc/releases/2022-12-01/uklwc.owl" \
--annotation-file metadata/uklwc.ttl \
--output build/uklwc.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/gcs.tsv | tr '#' '\n' > build/gcs.tsv
python3 src/create_xref_template.py build/gcs.tsv build/intermediate/gecko_index.tsv build/intermediate/gcs-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/gcs.tsv \
template \
--template build/intermediate/gcs-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/gcs.owl" \
--version-iri "https://purl.ihccglobal.org/gcs/releases/2022-12-01/gcs.owl" \
--annotation-file metadata/gcs.ttl \
--output build/gcs.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/hunt.tsv | tr '#' '\n' > build/hunt.tsv
python3 src/create_xref_template.py build/hunt.tsv build/intermediate/gecko_index.tsv build/intermediate/hunt-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/hunt.tsv \
template \
--template build/intermediate/hunt-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/hunt.owl" \
--version-iri "https://purl.ihccglobal.org/hunt/releases/2022-12-01/hunt.owl" \
--annotation-file metadata/hunt.ttl \
--output build/hunt.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/elsabrasil.tsv | tr '#' '\n' > build/elsabrasil.tsv
python3 src/create_xref_template.py build/elsabrasil.tsv build/intermediate/gecko_index.tsv build/intermediate/elsabrasil-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/elsabrasil.tsv \
template \
--template build/intermediate/elsabrasil-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/elsabrasil.owl" \
--version-iri "https://purl.ihccglobal.org/elsabrasil/releases/2022-12-01/elsabrasil.owl" \
--annotation-file metadata/elsabrasil.ttl \
--output build/elsabrasil.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/plco.tsv | tr '#' '\n' > build/plco.tsv
python3 src/create_xref_template.py build/plco.tsv build/intermediate/gecko_index.tsv build/intermediate/plco-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/plco.tsv \
template \
--template build/intermediate/plco-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/plco.owl" \
--version-iri "https://purl.ihccglobal.org/plco/releases/2022-12-01/plco.owl" \
--annotation-file metadata/plco.ttl \
--output build/plco.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/gs.tsv | tr '#' '\n' > build/gs.tsv
python3 src/create_xref_template.py build/gs.tsv build/intermediate/gecko_index.tsv build/intermediate/gs-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/gs.tsv \
template \
--template build/intermediate/gs-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/gs.owl" \
--version-iri "https://purl.ihccglobal.org/gs/releases/2022-12-01/gs.owl" \
--annotation-file metadata/gs.ttl \
--output build/gs.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/canpath.tsv | tr '#' '\n' > build/canpath.tsv
python3 src/create_xref_template.py build/canpath.tsv build/intermediate/gecko_index.tsv build/intermediate/canpath-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/canpath.tsv \
template \
--template build/intermediate/canpath-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/canpath.owl" \
--version-iri "https://purl.ihccglobal.org/canpath/releases/2022-12-01/canpath.owl" \
--annotation-file metadata/canpath.ttl \
--output build/canpath.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/koges.tsv | tr '#' '\n' > build/koges.tsv
python3 src/create_xref_template.py build/koges.tsv build/intermediate/gecko_index.tsv build/intermediate/koges-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/koges.tsv \
template \
--template build/intermediate/koges-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/koges.owl" \
--version-iri "https://purl.ihccglobal.org/koges/releases/2022-12-01/koges.owl" \
--annotation-file metadata/koges.ttl \
--output build/koges.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/saprin.tsv | tr '#' '\n' > build/saprin.tsv
python3 src/create_xref_template.py build/saprin.tsv build/intermediate/gecko_index.tsv build/intermediate/saprin-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/saprin.tsv \
template \
--template build/intermediate/saprin-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/saprin.owl" \
--version-iri "https://purl.ihccglobal.org/saprin/releases/2022-12-01/saprin.owl" \
--annotation-file metadata/saprin.ttl \
--output build/saprin.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/vz.tsv | tr '#' '\n' > build/vz.tsv
python3 src/create_xref_template.py build/vz.tsv build/intermediate/gecko_index.tsv build/intermediate/vz-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/vz.tsv \
template \
--template build/intermediate/vz-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/vz.owl" \
--version-iri "https://purl.ihccglobal.org/vz/releases/2022-12-01/vz.owl" \
--annotation-file metadata/vz.ttl \
--output build/vz.owl
sed -E '2s/^/ID	LABEL	C % SPLIT=|	A definition		A internal ID#	is-required;#/' templates/testthre.tsv | tr '#' '\n' > build/testthre.tsv
python3 src/create_xref_template.py build/testthre.tsv build/intermediate/gecko_index.tsv build/intermediate/testthre-xrefs.tsv
java -jar build/robot.jar --prefixes src/prefixes.json template --input build/intermediate/properties.owl \
--merge-before \
--template build/testthre.tsv \
template \
--template build/intermediate/testthre-xrefs.tsv \
--merge-before \
annotate --ontology-iri "https://purl.ihccglobal.org/testthre.owl" \
--version-iri "https://purl.ihccglobal.org/testthre/releases/2022-12-01/testthre.owl" \
--annotation-file metadata/testthre.ttl \
--output build/testthre.owl
cp build/chop.owl build/ge.owl build/uklwc.owl build/gcs.owl build/hunt.owl build/elsabrasil.owl build/plco.owl build/gs.owl build/canpath.owl build/koges.owl build/saprin.owl build/vz.owl build/testthre.owl data_dictionaries/
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/chop.owl --query src/queries/ihcc-mapping.sparql build/intermediate/chop-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/ge.owl --query src/queries/ihcc-mapping.sparql build/intermediate/ge-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/uklwc.owl --query src/queries/ihcc-mapping.sparql build/intermediate/uklwc-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/gcs.owl --query src/queries/ihcc-mapping.sparql build/intermediate/gcs-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/hunt.owl --query src/queries/ihcc-mapping.sparql build/intermediate/hunt-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/elsabrasil.owl --query src/queries/ihcc-mapping.sparql build/intermediate/elsabrasil-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/plco.owl --query src/queries/ihcc-mapping.sparql build/intermediate/plco-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/gs.owl --query src/queries/ihcc-mapping.sparql build/intermediate/gs-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/canpath.owl --query src/queries/ihcc-mapping.sparql build/intermediate/canpath-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/koges.owl --query src/queries/ihcc-mapping.sparql build/intermediate/koges-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/saprin.owl --query src/queries/ihcc-mapping.sparql build/intermediate/saprin-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/vz.owl --query src/queries/ihcc-mapping.sparql build/intermediate/vz-xrefs-sparql.csv
java -jar build/robot.jar --prefixes src/prefixes.json query --input build/testthre.owl --query src/queries/ihcc-mapping.sparql build/intermediate/testthre-xrefs-sparql.csv
python3 src/mapping-suggest/ihcc-zooma-dataset.py  -l build/intermediate/gecko-xrefs-sparql.csv  -l build/intermediate/chop-xrefs-sparql.csv  -l build/intermediate/ge-xrefs-sparql.csv  -l build/intermediate/uklwc-xrefs-sparql.csv  -l build/intermediate/gcs-xrefs-sparql.csv  -l build/intermediate/hunt-xrefs-sparql.csv  -l build/intermediate/elsabrasil-xrefs-sparql.csv  -l build/intermediate/plco-xrefs-sparql.csv  -l build/intermediate/gs-xrefs-sparql.csv  -l build/intermediate/canpath-xrefs-sparql.csv  -l build/intermediate/koges-xrefs-sparql.csv  -l build/intermediate/saprin-xrefs-sparql.csv  -l build/intermediate/vz-xrefs-sparql.csv  -l build/intermediate/testthre-xrefs-sparql.csv -w /workspace -o data/ihcc-mapping-suggestions-zooma.tsv
python3 src/prepare_ols_config.py  -m metadata/chop.ttl  -m metadata/ge.ttl  -m metadata/uklwc.ttl  -m metadata/gcs.ttl  -m metadata/hunt.ttl  -m metadata/elsabrasil.ttl  -m metadata/plco.ttl  -m metadata/gs.ttl  -m metadata/canpath.ttl  -m metadata/koges.ttl  -m metadata/saprin.ttl  -m metadata/vz.ttl  -m metadata/testthre.ttl -o data/ols-config.yaml
Preparing OLS config successful
rm -rf build/chop.db
sqlite3 build/chop.db < build/prefixes.sql
./build/rdftab build/chop.db < build/chop.owl
rm -rf build/ge.db
sqlite3 build/ge.db < build/prefixes.sql
./build/rdftab build/ge.db < build/ge.owl
rm -rf build/uklwc.db
sqlite3 build/uklwc.db < build/prefixes.sql
./build/rdftab build/uklwc.db < build/uklwc.owl
rm -rf build/gcs.db
sqlite3 build/gcs.db < build/prefixes.sql
./build/rdftab build/gcs.db < build/gcs.owl
rm -rf build/hunt.db
sqlite3 build/hunt.db < build/prefixes.sql
./build/rdftab build/hunt.db < build/hunt.owl
rm -rf build/elsabrasil.db
sqlite3 build/elsabrasil.db < build/prefixes.sql
./build/rdftab build/elsabrasil.db < build/elsabrasil.owl
rm -rf build/plco.db
sqlite3 build/plco.db < build/prefixes.sql
./build/rdftab build/plco.db < build/plco.owl
rm -rf build/gs.db
sqlite3 build/gs.db < build/prefixes.sql
./build/rdftab build/gs.db < build/gs.owl
rm -rf build/canpath.db
sqlite3 build/canpath.db < build/prefixes.sql
./build/rdftab build/canpath.db < build/canpath.owl
rm -rf build/koges.db
sqlite3 build/koges.db < build/prefixes.sql
./build/rdftab build/koges.db < build/koges.owl
rm -rf build/saprin.db
sqlite3 build/saprin.db < build/prefixes.sql
./build/rdftab build/saprin.db < build/saprin.owl
rm -rf build/vz.db
sqlite3 build/vz.db < build/prefixes.sql
./build/rdftab build/vz.db < build/vz.owl
rm -rf build/testthre.db
sqlite3 build/testthre.db < build/prefixes.sql
./build/rdftab build/testthre.db < build/testthre.owl
python3 -m gizmos.export \
-d build/chop.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'CHOP:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/chop.html
WARNING:root:'rdfs:subClassOf' does not exist in database
python3 -m gizmos.export \
-d build/ge.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'GE:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/ge.html
python3 -m gizmos.export \
-d build/uklwc.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'UKLWC:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/uklwc.html
python3 -m gizmos.export \
-d build/gcs.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'GCS:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/gcs.html
python3 -m gizmos.export \
-d build/hunt.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'HUNT:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/hunt.html
python3 -m gizmos.export \
-d build/elsabrasil.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'ELSABRASIL:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/elsabrasil.html
WARNING:root:'rdfs:subClassOf' does not exist in database
python3 -m gizmos.export \
-d build/plco.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'PLCO:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/plco.html
python3 -m gizmos.export \
-d build/gs.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'GS:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/gs.html
python3 -m gizmos.export \
-d build/canpath.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'CANPATH:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/canpath.html
WARNING:root:'rdfs:subClassOf' does not exist in database
python3 -m gizmos.export \
-d build/koges.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'KOGES:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/koges.html
python3 -m gizmos.export \
-d build/saprin.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'SAPRIN:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/saprin.html
python3 -m gizmos.export \
-d build/vz.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'VZ:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/vz.html
WARNING:root:'rdfs:subClassOf' does not exist in database
python3 -m gizmos.export \
-d build/testthre.db \
-f html \
-p CURIE -p label -p rdfs:subClassOf -p definition -p oboInOwl:hasDbXref -p IHCC:internal-id \
-V CURIE \
-w "stanza LIKE 'TESTTHRE:%'" | \
sed 's/rdfs:subClassOf/parent/g' | \
sed 's/oboInOwl:hasDbXref/GECKO category/g' | \
sed 's/IHCC:internal-id/internal ID/g' > build/testthre.html
python3 src/create_index.py src/index.html.jinja2 data/metadata.json build/index.html
python3 src/convert_metadata.py data/metadata.json build/member_cohorts.csv

Success