Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
0d044a8
move to blazegraph
anitacaron Jun 27, 2023
8a18679
create blazegraph dockerfile
anitacaron Jul 18, 2023
7627563
update blazegraph docker image
anitacaron Jul 21, 2023
bd03c53
create RWStore.properties
anitacaron Aug 9, 2023
4753b12
update upload-triplestore docker image
anitacaron Aug 9, 2023
2b1049e
change data to ttl to load into blazegraph
anitacaron Aug 9, 2023
9991b3b
update blazegraph config and dataloader
anitacaron Aug 11, 2023
10b7fe6
update obo context
anitacaron Sep 11, 2023
2dc246d
update data volume directory
anitacaron Sep 11, 2023
8a21d5a
update registry parser
anitacaron Sep 11, 2023
b164840
search entities endpoint by confidence (min and max)
anitacaron Sep 11, 2023
4576c1d
enable search mapping_justification by curie
anitacaron Sep 12, 2023
a50660f
fix lint errors
anitacaron Sep 12, 2023
67b14c3
fix lint errors
anitacaron Sep 12, 2023
1c52908
add PyLD to poetry
anitacaron Sep 12, 2023
8793106
fix lint error in registry_parser
anitacaron Sep 12, 2023
49602f9
fix lint errors in utils
anitacaron Sep 12, 2023
36c0f4e
add mapping_justification_curie to response in mappings endpoints
anitacaron Sep 21, 2023
a7811a1
update packages
anitacaron Nov 16, 2023
a35b010
update query method
anitacaron Nov 16, 2023
c5d57d5
Merge pull request #84 from mapping-commons/anitacaron/issue35
Nov 16, 2023
f9add0f
remove rdf4j config files
anitacaron Nov 17, 2023
9995994
move data to .gitignore
anitacaron Nov 17, 2023
8330801
move config files to config folder
anitacaron Nov 17, 2023
e2c4d7b
improve query
anitacaron Nov 17, 2023
71e2e41
parse local_name
anitacaron Nov 17, 2023
6c9ea4e
update obo context
anitacaron Dec 1, 2023
8c381b7
add rule to update context
anitacaron Dec 1, 2023
be0cf4a
clean docker-compose file
anitacaron Dec 1, 2023
94e65d4
use the env variable `JAVA_OPTS` when running blazegraph and remove n…
anitacaron Jun 27, 2024
3c57ef9
remove unneeded code in the `api.dockerfile` that was moved to `docke…
anitacaron Jun 27, 2024
890c3ff
update sssom dependency to fix numpy `np.NAN` issue
anitacaron Jun 27, 2024
d9163a9
improve query results efficiency for the `entities`, `mappings` and `…
anitacaron Jun 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
.vscode
__pycache__
.env
/data/mappings.ttl
/data/mappings.jsonld
/data/registry.yml
/data/*
poetry.lock
6 changes: 5 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ REGISTRY_URL = https://raw.githubusercontent.com/mapping-commons/mh_mapping_init
$(PWD)/data/registry.yml:
wget $(REGISTRY_URL) -O $@

jsonld_from_registry: $(PWD)/data/registry.yml
ttl_from_registry: $(PWD)/data/registry.yml
cd src && python3 registry_parser.py $<

deploy_api:
Expand All @@ -19,3 +19,7 @@ rebuild-%:
build-sssom-api:
docker-compose up --build

OBO_CONTEXT_URL = https://raw.githubusercontent.com/biopragmatics/bioregistry/main/exports/contexts/obo.context.jsonld
.PHONY: update_obo_context
update_obo_context:
wget $(OBO_CONTEXT_URL) -O resources/obo.context.jsonld
4 changes: 1 addition & 3 deletions api.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,4 @@ RUN pip install poetry

RUN poetry config virtualenvs.create false

RUN poetry install

# CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
RUN poetry install
22 changes: 22 additions & 0 deletions blazegraph.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
FROM ubuntu:20.04
WORKDIR /tools

# Install base tools from Ubuntu.
RUN apt-get update && \
DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
wget \
curl \
openjdk-11-jdk \
ca-certificates \
ca-certificates-java

ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/
RUN export JAVA_HOME

###### Blazegraph #####
ENV JAVA_OPTS -Xmx16G
RUN wget -nv https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar

CMD java -server $JAVA_OPTS -Dcom.bigdata.rdf.sail.webapp.ConfigParams.propertyFile=/RWStore.properties -Dcom.bigdata.util.config.LogUtil=/log4.properties -jar blazegraph.jar


45 changes: 45 additions & 0 deletions config/RWStore.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#
# Note: These options are applied when the journal and the triple store are
# first created.

##
## Journal options.
##

# The backing file. This contains all your data. You want to put this someplace
# safe. The default locator will wind up in the directory from which you start
# your servlet container.
com.bigdata.journal.AbstractJournal.file=blazegraph-mappings.jnl

# The persistence engine. Use 'Disk' for the WORM or 'DiskRW' for the RWStore.
com.bigdata.journal.AbstractJournal.bufferMode=DiskRW

# Setup for the RWStore recycler rather than session protection.
com.bigdata.service.AbstractTransactionService.minReleaseAge=1

com.bigdata.btree.writeRetentionQueue.capacity=4000
com.bigdata.btree.BTree.branchingFactor=128

# 200M initial extent.
com.bigdata.journal.AbstractJournal.initialExtent=209715200
com.bigdata.journal.AbstractJournal.maximumExtent=209715200

##
## Setup for triples mode without the full text index.
##
com.bigdata.rdf.sail.truthMaintenance=false
com.bigdata.rdf.store.AbstractTripleStore.quads=false
com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=false
com.bigdata.rdf.store.AbstractTripleStore.textIndex=false
com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms

# Bump up the branching factor for the lexicon indices on the default kb.
com.bigdata.namespace.kb.lex.com.bigdata.btree.BTree.branchingFactor=400

# Bump up the branching factor for the statement indices on the default kb.
com.bigdata.namespace.kb.spo.com.bigdata.btree.BTree.branchingFactor=1024

com.bigdata.search.FullTextIndex.analyzerFactoryClass=com.bigdata.search.ConfigurableAnalyzerFactory
com.bigdata.search.ConfigurableAnalyzerFactory.analyzer.eng.analyzerClass=org.apache.lucene.analysis.standard.StandardAnalyzer
com.bigdata.search.ConfigurableAnalyzerFactory.analyzer.eng.stopwords=none
com.bigdata.search.ConfigurableAnalyzerFactory.analyzer._.like=eng
24 changes: 24 additions & 0 deletions config/dataloader.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<!-- Default Graph URI (Optional - Required for quads mode namespace) -->
<entry key="defaultGraph">kb</entry>
<!-- The configuration file for the database instance. It must be readable by the web application. -->
<entry key="propertyFile">/RWStore.properties</entry>
<!-- The namespace of the KB instance. Defaults to kb. -->
<entry key="namespace">kb</entry>
<!-- Zero or more files or directories containing the data to be loaded.
This should be a comma delimited list. The files must be readable by the web application. -->
<entry key="fileOrDirs">/data</entry>
<!-- RDF Format (Default is rdf/xml) -->
<entry key="format">x-turtle</entry>
<!-- Suppress all stdout messages (Optional) -->
<entry key="quiet">false</entry>
<!-- Show additional messages detailing the load performance. (Optional) -->
<entry key="verbose">100</entry>
<!-- Compute the RDF(S)+ closure. (Optional) -->
<entry key="closure">false</entry>
<!-- Files will be renamed to either .good or .fail as they are processed.
The files will remain in the same directory. -->
<entry key="durableQueues">true</entry>
</properties>
Binary file removed data/ols/ado.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/aeo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/aero.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/afo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/agro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/aism.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/amphx.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/apo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/apollo_sv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/aro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bcgo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bcio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bfo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bila.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bspo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/bto.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/caro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ccf.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cdao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cdno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ceph.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cheminf.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/chiro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/chmo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cl.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/clao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/clo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/clyh.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cmo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cmpo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cob.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/colao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/covoc.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cpont.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cryoem.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/cteno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dc.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dcterms.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ddanat.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ddpheno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dicom.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dideo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/disdriv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/doid.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dpo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/dron.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/duo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ecao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/eco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ecocore.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ecto.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/edam.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/efo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ehdaa2.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/emap.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/emapa.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/enm.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ensemblglossary.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/envo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/epio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/eupath.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/exo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fbbi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fbbt.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fbcv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fbdv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fbsp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fideo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fix.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/flopo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fma.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fobi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/foodon.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fovt.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/fypo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/gecko.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/geno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/geo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/gexo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/gno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/go.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/gsso.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hancestro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hcao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hom.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hsapdv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/hso.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/htn.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/iao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/iceo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ico.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ido.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/idocovid19.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/idomal.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ino.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/kisao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/labo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/lepao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ma.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mappings.jsonld.gz
Binary file not shown.
Binary file removed data/ols/maxo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mf.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mfmo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mfoem.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mfomd.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/miapa.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/micro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/miro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mmo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mmusdv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mod.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file not shown.
Binary file removed data/ols/mondo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mondo.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mop.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mp_hp_mgi_all.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mpath.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mpio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/mro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ms.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/msio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/nbo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ncit.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ncro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ngbo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/nmrcv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/nomen.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/oae.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/oarcs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/oba.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/obcs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/obi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/obib.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/oboinowl.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ogg.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ogms.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ogsf.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ohd.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ohmi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ohpi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/olatdv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/om.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/omiabis.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/omit.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/omo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/omp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/omrse.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/one.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ons.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ontoavida.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ontoneo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/oostt.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/opl.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/opmi.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ordo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ornaseq.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/orth.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ovae.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/owl.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pato.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pcl.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pdro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pdumdv.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/peco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/phipo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/plana.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/planp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/po.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/poro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ppo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pr.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/proco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/prov.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/psdo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pso.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/pw.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rbo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rdfs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/reproduceme.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/reto.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rex.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rexo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/ro.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/rxno.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/sbo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/scdo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/schemaorg_http.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/schemaorg_https.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/sdgio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/sepio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/shareloc.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/sibo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/sio.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/skos.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/snomed.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/so.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/spd.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/srao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/stato.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/swo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/symp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/t4fs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/tads.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/tao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/taxrank.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/tgma.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/to.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/trans.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/tto.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/txpo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/uberon.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/uo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/upa.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/vbo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/vo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/vsao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/vt.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/vto.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/wbbt.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/wbls.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/wbphenotype.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/xao.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/xco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/xlmod.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/xpo.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/zeco.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/zfa.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/zfs.ols.sssom.tsv.jsonld.gz
Binary file not shown.
Binary file removed data/ols/zp.ols.sssom.tsv.jsonld.gz
Binary file not shown.
23 changes: 16 additions & 7 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,22 @@ services:
- triplestore
- upload-triplestore
environment:
- SPARQL_ENDPOINT=http://triplestore:8080/rdf4j-server/repositories/sssom
- SPARQL_ENDPOINT=http://triplestore:9999/blazegraph/namespace/kb/sparql
triplestore:
image: eclipse/rdf4j-workbench:latest
image: anitacaron/blazegraph:v0.4
environment:
- JAVA_OPTS=-Xms10g -Xmx20g
- RDF4J_DATA=/data
- JAVA_OPTS=-Xmx10G
ports:
- 8080:8080
- 9999:9999
volumes:
- ./data:/data
- ./config/RWStore.properties:/RWStore.properties
- ./config/log4j.properties:/log4j.properties
healthcheck:
test: ["CMD", "curl", "-f", "http://triplestore:9999/"]
interval: 3s
timeout: 5s
retries: 3
upload-triplestore:
build:
context: .
Expand All @@ -32,6 +40,7 @@ services:
links:
- triplestore
environment:
- SERVER=http://triplestore:8080
- SERVER=http://triplestore:9999/blazegraph
volumes:
- ./data:/data
- ./data:/data
- ./config/RWStore.properties:/RWStore.properties
51 changes: 30 additions & 21 deletions process.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,45 +7,54 @@ echo "Start: upload-triplestore"
echo "TIME:"
date

SETUP=${WORKSPACE}/rdf4j_sssom.txt
RDF4J=/opt/eclipse-rdf4j-${RDF4J_VERSION}
RDF4JSERVER=${SERVER}/rdf4j-server
DATA=/data/ols
# SETUP=${WORKSPACE}/rdf4j_sssom.txt
#BLAZEGRAPH_UID=/opt/eclipse-rdf4j-${RDF4J_VERSION}
BLAZEGRAPHSERVER=${SERVER}/dataloader
DATA=/data
DATALOADER=/opt/SSSOM/

if [ `ls $DATA/*.jsonld.gz | wc -l` -lt 1 ]; then echo "ERROR: No data in data directory! Aborting.. " && exit 1; fi
#if [ `ls $DATA/*.jsonld.gz | wc -l` -lt 1 ]; then echo "ERROR: No data in data directory! Aborting.. " && exit 1; fi

echo 'Waiting for RDF4J server..'
until $(curl --output /dev/null --silent --head --fail ${RDF4JSERVER}); do
echo 'Waiting for BLAZEGRAPH WORKBENCH..'
until $(curl --output /dev/null --silent --head --fail ${SERVER}); do
printf '.'
sleep 5
done

echo "connect "${RDF4JSERVER}|cat - ${SETUP} > /tmp/out && mv /tmp/out ${SETUP}
cat ${SETUP}
cat ${SETUP} | sh ${RDF4J}/bin/console.sh
# echo "connect "${RDF4JSERVER}|cat - ${SETUP} > /tmp/out && mv /tmp/out ${SETUP}
# cat ${SETUP}
# cat ${SETUP} | sh ${RDF4J}/bin/console.sh

ls -lh $DATA


echo "TIME:"
date

cd $DATA
cd $DATALOADER
echo $BLAZEGRAPHSERVER

echo "curl -X POST --data-binary @dataloader.xml --header 'Content-Type:application/xml' http://localhost:9999/blazegraph/dataloader"
curl POST -v --data-binary @dataloader.xml --header 'Content-Type:application/xml' ${BLAZEGRAPHSERVER}



# The following for loop writes the load commands into the RDF4J setup script
for i in *.jsonld.gz; do
[ -f "$i" ] || break
#arg="load "$DATA/$i" into ns:"$i
echo $i
# for i in *.jsonld.gz; do
# [ -f "$i" ] || break
# #arg="load "$DATA/$i" into ns:"$i
# echo $i
#awk -v line="$arg" '/open vfb/ { print; print line; next }1' $WS/rdf4j.txt > $WS/tmp.txt
#cp $WS/tmp.txt $WS/rdf4j.txt
# URI="%3Chttp%3A%2F%2Fvirtualflybrain.org%2Fdata%2FVFB%2FOWL%2F${i}%3E"
echo "curl -v --retry 5 --retry-delay 10 -X POST -H \"Content-type: application/ld+json\" --data-binary @$i ${RDF4JSERVER}/repositories/sssom/statements?context=null"
curl -v --retry 5 --retry-delay 10 -X POST -H "Content-type: application/ld+json" --data-binary @$i ${RDF4JSERVER}/repositories/sssom/statements?context=null || exit 1
echo "TIME:"
date
sleep 5
done
# echo "curl -v --retry 5 --retry-delay 10 -X POST -H \"Content-type: application/ld+json\" --data-binary @$i ${RDF4JSERVER}/repositories/sssom/statements?context=null"
# echo "curl -X POST --data-binary @dataloader.xml --header 'Content-Type:application/xml' http://localhost:9999/bigdata/dataloader"
# curl -v --retry 5 --retry-delay 10 -X POST -H "Content-type: application/ld+json" --data-binary @$i ${RDF4JSERVER}/repositories/sssom/statements?context=null || exit 1
# curl -X POST --data-binary @dataloader.xml --header 'Content-Type:application/xml' ${BLAZEGRAPHSERVER}
# echo "TIME:"
# date
# sleep 5
# done


echo "End: upload-triplestore"
Expand Down
7 changes: 4 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@ python = "^3.9"
fastapi = "^0.95.0"
uvicorn = {extras = ["standard"], version = "^0.21.1"}
pydantic = {extras = ["dotenv"], version = "^1.10.7"}
oaklib = "0.1.71"
sssom-schema = "^v0.9.4"
sssom = "^0.3.26"
sssom-schema = "^0.15.0"
sssom = "0.4.10"
toolz = "^0.12.0"
pyld = "^2.0.3"
oaklib = "^0.5.21"

[tool.poetry.group.dev.dependencies]
pytest = "^7.2.2"
Expand Down
14 changes: 0 additions & 14 deletions rdf4j_sssom.txt

This file was deleted.

Loading