dump (draft) BT release from BC db and publish

jobs/release_fptwg

@@ -28,7 +28,7 @@ ogr2ogr \
             aggregated_crossings_id,
             stream_crossing_id,
             dam_id,
-            user_barrier_anthropogenic_id,
+            user_crossing_misc_id,
             modelled_crossing_id,
             crossing_source,
             crossing_feature_type,
@@ -250,4 +250,208 @@ sozip \
   freshwater_fish_habitat_accessibility_MODEL.gpkg.zip \
   freshwater_fish_habitat_accessibility_MODEL.gpkg
 
-aws s3 cp freshwater_fish_habitat_accessibility_MODEL.gpkg.zip s3://bchamp --acl public-read
+aws s3 cp freshwater_fish_habitat_accessibility_MODEL.gpkg.zip s3://bchamp --acl public-read
+
+
+# release a draft/undocumented full dump of salmon/steelhead/bull trout/wct
+ogr2ogr \
+  -f Parquet \
+  bcfishpass_streams.parquet \
+  PG:$DATABASE_URL \
+  --debug ON \
+  -lco COMPRESSION=ZSTD \
+  -lco COMPRESSION_LEVEL=15 \
+  -lco SORT_BY_BBOX=YES \
+  -nln bcfishpass_streams \
+  -sql "SELECT
+  segmented_stream_id,
+  linear_feature_id,
+  edge_type,
+  blue_line_key,
+  watershed_key,
+  watershed_group_code,
+  downstream_route_measure,
+  length_metre,
+  waterbody_key,
+  wscode::text as wscode,
+  localcode::text as localcode,
+  gnis_name,
+  stream_order,
+  stream_magnitude,
+  gradient,
+  feature_code,
+  upstream_route_measure,
+  upstream_area_ha,
+  stream_order_parent,
+  stream_order_max,
+  map_upstream,
+  channel_width,
+  mad_m3s,
+  barriers_anthropogenic_dnstr,
+  barriers_pscis_dnstr,
+  barriers_dams_dnstr,
+  barriers_dams_hydro_dnstr,
+  barriers_bt_dnstr,
+  crossings_dnstr,
+  dam_dnstr_ind,
+  dam_hydro_dnstr_ind,
+  remediated_dnstr_ind,
+  observation_key_upstr,
+  obsrvtn_species_codes_upstr,
+  species_codes_dnstr,
+  access_bt,
+  access_ch,
+  access_cm,
+  access_co,
+  access_pk,
+  access_sk,
+  access_st,
+  access_wct,
+  access_salmon,
+  spawning_bt,
+  spawning_ch,
+  spawning_cm,
+  spawning_co,
+  spawning_pk,
+  spawning_sk,
+  spawning_st,
+  spawning_wct,
+  rearing_bt,
+  rearing_ch,
+  rearing_co,
+  rearing_sk,
+  rearing_st,
+  rearing_wct,
+  geom
+FROM bcfishpass.streams_vw
+WHERE 
+  access_bt > 0 OR
+  access_ch > 0 OR
+  access_cm > 0 OR
+  access_co > 0 OR
+  access_pk > 0 OR
+  access_sk > 0 OR
+  access_st > 0 OR
+  access_wct > 0"
+
+ogr2ogr \
+    -f GPKG \
+    bcfishpass_crossings.gpkg.zip \
+    PG:$DATABASE_URL \
+    -nln bcfishpass_crossings \
+    -nlt PointZM \
+    -sql "select * FROM bcfishpass.crossings_vw"
+
+# dump also a basic top crossings report
+# this is just a draft for now, aggregated spawn/rear columns
+# should be added to crossings_vw for better ranking
+psql $DATABASE_URL -c "SELECT 
+ aggregated_crossings_id,
+ stream_crossing_id,
+ dam_id,
+ user_crossing_misc_id,
+ modelled_crossing_id,
+ crossing_source,
+ crossing_feature_type,
+ pscis_status,
+ crossing_type_code,
+ crossing_subtype_code,
+ modelled_crossing_type_source,
+ barrier_status,
+ pscis_road_name,
+ pscis_stream_name,
+ pscis_assessment_comment,
+ pscis_assessment_date,
+ pscis_final_score,
+ transport_line_structured_name_1,
+ transport_line_type_description,
+ transport_line_surface_description,
+ ften_forest_file_id,
+ ften_road_section_id,
+ ften_file_type_description,
+ ften_client_number,
+ ften_client_name,
+ ften_life_cycle_status_code,
+ ften_map_label,
+ rail_track_name,
+ rail_owner_name,
+ rail_operator_english_name,
+ ogc_proponent,
+ dam_name,
+ dam_height,
+ dam_owner,
+ dam_use,
+ dam_operating_status,
+ linear_feature_id,
+ blue_line_key,
+ watershed_key,
+ downstream_route_measure,
+ wscode,
+ localcode,
+ watershed_group_code,
+ gnis_stream_name,
+ stream_order,
+ stream_magnitude,
+ observedspp_dnstr,
+ observedspp_upstr,
+ crossings_dnstr,
+ barriers_anthropogenic_dnstr_count,
+ barriers_anthropogenic_upstr_count,
+ barriers_anthropogenic_ch_cm_co_pk_sk_upstr_count,
+ barriers_anthropogenic_st_upstr_count,
+ bt_spawning_km,
+ bt_rearing_km,
+ bt_spawningrearing_km,
+ bt_spawning_belowupstrbarriers_km,
+ bt_rearing_belowupstrbarriers_km,
+ bt_spawningrearing_belowupstrbarriers_km,
+ ch_spawning_km,
+ ch_rearing_km,
+ ch_spawningrearing_km,
+ ch_spawning_belowupstrbarriers_km,
+ ch_rearing_belowupstrbarriers_km,
+ ch_spawningrearing_belowupstrbarriers_km,
+ cm_spawning_km,
+ cm_spawning_belowupstrbarriers_km,
+ co_spawning_km,
+ co_rearing_km,
+ co_spawningrearing_km,
+ co_rearing_ha,
+ co_spawning_belowupstrbarriers_km,
+ co_rearing_belowupstrbarriers_km,
+ co_rearing_belowupstrbarriers_ha,
+ co_spawningrearing_belowupstrbarriers_km,
+ pk_spawning_km,
+ pk_spawning_belowupstrbarriers_km,
+ sk_spawning_km,
+ sk_rearing_km,
+ sk_rearing_ha,
+ sk_spawningrearing_km,
+ sk_spawning_belowupstrbarriers_km,
+ sk_rearing_belowupstrbarriers_km,
+ sk_rearing_belowupstrbarriers_ha,
+ sk_spawningrearing_belowupstrbarriers_km,
+ st_spawning_km,
+ st_rearing_km,
+ st_spawningrearing_km,
+ st_spawning_belowupstrbarriers_km,
+ st_rearing_belowupstrbarriers_km,
+ st_spawningrearing_belowupstrbarriers_km,
+ salmon_spawningrearing_km,
+ salmon_spawningrearing_belowupstrbarriers_km,
+ salmonsteelhead_spawningrearing_km,
+ salmonsteelhead_spawningrearing_belowupstrbarriers_km,
+ wct_spawning_km,
+ wct_rearing_km,
+ wct_spawningrearing_km,
+ wct_spawning_belowupstrbarriers_km,
+ wct_rearing_belowupstrbarriers_km,
+ wct_spawningrearing_belowupstrbarriers_km
+FROM bcfishpass.crossings_vw
+WHERE barrier_status in ('BARRIER','POTENTIAL')
+AND salmonsteelhead_spawningrearing_belowupstrbarriers_km > 0
+order by salmonsteelhead_spawningrearing_belowupstrbarriers_km desc
+limit 500" --csv > barriers_salmonsteelhead.csv    
+
+aws s3 cp bcfishpass_streams.parquet s3://bchamp --acl public-read
+aws s3 cp bcfishpass_crossings.parquet s3://bchamp --acl public-read