Bump django from 1.11.27 to 2.2.24

Makefile

@@ -25,6 +25,7 @@ DEV_COMMON_EXPRTS=export OUT_IMG=${DEV_OUT_IMG};  export BASE_IMG=${UI_BASE_IMG}
 
 # Base #
 base-build:
+	echo ${BASE_IMG}
 	docker build . -f build/docker/base/Dockerfile --build-arg BASE_IMG=${BASE_IMG} -t ${UI_BASE_IMG}
 
 base-run:
@@ -43,7 +44,7 @@ dev-build-no-rcv:
 dev-build: base-build dev-build-no-rcv
 
 # Start the UI
-dev-up: 
+dev-up: base-build
 	(${DEV_COMMON_EXPRTS}; $(docker_compose_dev) up -d --build)
 
 # Start without rebuilding the Docker image
@@ -110,7 +111,7 @@ create-odc-db:
 	--name=odc-db \
 	--network="odc" \
 	-v odc-db-vol:/var/lib/postgresql/data \
-	postgis/postgis:10-2.5
+	postgis/postgis:10-2.5 \
 	-N 1000 \
 	-B 2048MB
 	# postgres:10-alpine

apps/cloud_coverage/tasks.py

@@ -55,7 +55,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = CloudCoverageTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),

apps/coastal_change/tasks.py

@@ -11,7 +11,7 @@
 from utils.data_cube_utilities.data_access_api import DataAccessApi
 from utils.data_cube_utilities.dc_coastal_change import compute_coastal_change, mask_mosaic_with_coastal_change, mask_mosaic_with_coastlines
 from utils.data_cube_utilities.dc_utilities import (create_cfmask_clean_mask, create_bit_mask, write_geotiff_from_xr,
-                                                    write_png_from_xr, add_timestamp_data_to_xr, clear_attrs)
+                                                    write_png_from_xr, add_timestamp_data_to_xr, clear_attrs, convert_range)
 from utils.data_cube_utilities.dc_chunker import (create_geographic_chunks, group_datetimes_by_year,
                                                   combine_geographic_chunks)
 from apps.dc_algorithm.utils import create_2d_plot, _get_datetime_range_containing
@@ -56,7 +56,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = CoastalChangeTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (datetime(task.time_start, 1, 1), datetime(task.time_end, 12, 31)),
         'longitude': (task.longitude_min, task.longitude_max),
@@ -140,7 +139,7 @@ def perform_task_chunking(self, parameters, task_id=None):
         longitude=parameters['longitude'],
         latitude=parameters['latitude'],
         geographic_chunk_size=task_chunk_sizing['geographic'])
-
+    
     grouped_dates = group_datetimes_by_year(dates)
     # we need to pair these with the first year - subsequent years.
     time_chunks = None
@@ -281,6 +280,25 @@ def _compute_mosaic(time):
 
     metadata = {**old_metadata, **new_metadata}
 
+    # Ensure data variables have the range of Landsat Collection 1 Level 2
+    # since the color scales are tailored for that dataset.
+    platform = task.satellite.platform
+    collection = task.satellite.collection
+    level = task.satellite.level
+    mosaics = []
+    for data in [old_mosaic, new_mosaic]:
+        if collection != 'c1':
+            old_dataset = data
+            drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
+            data = \
+                convert_range(data.drop_vars(drop_vars), from_platform=platform, 
+                            from_collection=collection, from_level=level,
+                            to_platform=platform, to_collection='c1', to_level='l2')
+            for drop_var in drop_vars:
+                data[drop_var] = old_dataset[drop_var]
+        mosaics.append(data)
+    old_mosaic, new_mosaic = mosaics
+
     output_product = compute_coastal_change(old_mosaic, new_mosaic, no_data=task.satellite.no_data_value)
 
     if check_cancel_task(self, task): return

apps/dc_algorithm/management/commands/band_math_app/tasks.py

@@ -97,7 +97,6 @@ def parse_parameters_from_task(task_id=None):
     task = BandMathTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),

apps/dc_algorithm/models/application_models.py

@@ -31,6 +31,8 @@
 from utils.data_cube_utilities.clean_mask import landsat_qa_clean_mask
 from utils.data_cube_utilities.dc_mosaic import (ls5_unpack_qa, ls7_unpack_qa, ls8_unpack_qa)
 
+# TODO: This should be called `Datasource` or something similar - not Satellite.
+#       These could be drones or any other sources of geospatial data.
 class Satellite(models.Model):
     """Stores a satellite that exists in the Data Cube
 
@@ -50,20 +52,23 @@ class Satellite(models.Model):
         no_data_value: No data value to be used for all outputs/masking functionality.
 
     """
-
+    # TODO: This name needs to be changed. The datacube platforms are in `platform`.
+    #       This is used as a short name (`name` being the long name)
     datacube_platform = models.CharField(
         help_text="This should correspond with a Data Cube platform. Combinations should be comma seperated with no spaces, e.g. LANDSAT_7,LANDSAT_8",
         max_length=100)
-    name = models.CharField(max_length=100)
+    name = models.CharField(
+        help_text="This is the full name of this satellite.",
+        max_length=100)
 
     # product_prefix = models.CharField(
     #     max_length=250,
     #     help_text="Products are loaded by name with the naming convention product_prefix+area_id, e.g. ls5_ledaps_{vietnam,colombia,australia}, \
     #                s1a_gamma0_vietnam. For combined products, prefixes should be comma seperated with no spaces in the order of the datacube_platform."
     # )
 
-    date_min = models.DateField('date_min', default=timezone.now) #datetime.date.today)
-    date_max = models.DateField('date_max', default=timezone.now) #datetime.date.today)
+    date_min = models.DateField('date_min', default=timezone.now)
+    date_max = models.DateField('date_max', default=timezone.now)
 
     data_min = models.FloatField(
         help_text="Define the minimum of the valid range of this dataset. This is used for image creation/scaling.",
@@ -82,7 +87,8 @@ class Satellite(models.Model):
 
     platform = models.CharField(
         help_text='The platform associated with this satellite. ' \
-                  'Examples include "LANDSAT_5", "LANDSAT_7", or "LANDSAT_8"',
+                  'Examples include "LANDSAT_5", "LANDSAT_7", or "LANDSAT_8". ' \
+                  'Combinations should be comma seperated with no spaces, e.g. LANDSAT_7,LANDSAT_8.',
         default='',
         max_length=250)
 
@@ -141,7 +147,7 @@ def is_combined_product(self):
         return len(self.datacube_platform.split(",")) > 1
 
     def get_platforms(self):
-        return self.datacube_platform.split(",")
+        return self.platform.split(",")
 
     def get_products(self, area_id):
         from apps.dc_algorithm.models.application_models \

apps/fractional_cover/tasks.py

@@ -236,10 +236,8 @@ def start_chunk_processing(self, chunk_details, task_id=None):
     # to process, also considering intermediate chunks to be combined.
     num_scenes = len(geographic_chunks) * sum([len(time_chunk) for time_chunk in time_chunks])
     # recombine_time_chunks() and process_band_math() scenes:
-    # num_scn_per_chk * len(time_chunks) * len(geographic_chunks)
     num_scn_per_chk = round(num_scenes / (len(time_chunks) * len(geographic_chunks)))
     # recombine_geographic_chunks() and create_output_products() scenes:
-    # num_scn_per_chk_geo * len(geographic_chunks)
     num_scn_per_chk_geo = round(num_scenes / len(geographic_chunks))
     # Scene processing progress is tracked in: processing_task(), recombine_time_chunks(),
     # and process_band_math(). Scenes in process_band_math() are counted twice

apps/ndvi_anomaly/tasks.py

@@ -56,7 +56,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = NdviAnomalyTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),

apps/slip/tasks.py

@@ -66,7 +66,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = SlipTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),

apps/spectral_anomaly/tasks.py

@@ -85,7 +85,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = SpectralAnomalyTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'baseline_time': (task.baseline_time_start, task.baseline_time_end),
@@ -338,7 +337,10 @@ def processing_task(self,
         # Compute the spectral index for the composite.
         spec_ind_params = dict()
         if spectral_index == 'fractional_cover':
-            spec_ind_params = dict(clean_mask=composite_clean_mask, no_data=no_data_value)
+            platform = task.satellite.platform
+            collection = task.satellite.collection
+            spec_ind_params = dict(clean_mask=composite_clean_mask, no_data=no_data_value,
+                                   platform=platform, collection=collection)
         spec_ind_result = spectral_indices_function_map[spectral_index](composite, **spec_ind_params)
         if spectral_index in ['ndvi', 'ndbi', 'ndwi', 'evi']:
             composite[spectral_index] = spec_ind_result

apps/spectral_indices/tasks.py

@@ -114,7 +114,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = SpectralIndicesTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),
@@ -413,6 +412,22 @@ def _apply_band_math(dataset):
         return None
 
     dataset = xr.open_dataset(chunk[0]).load()
+
+    # Ensure data variables have the range of Landsat Collection 1 Level 2
+    # since the color scales are tailored for that dataset.
+    platform = task.satellite.platform
+    collection = task.satellite.collection
+    level = task.satellite.level
+    if collection != 'c1':
+        old_dataset = dataset
+        drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
+        dataset = \
+            convert_range(old_dataset.drop_vars(drop_vars), from_platform=platform, 
+                          from_collection=collection, from_level=level,
+                          to_platform=platform, to_collection='c1', to_level='l2')
+        for drop_var in drop_vars:
+            dataset[drop_var] = old_dataset[drop_var]
+
     dataset['band_math'] = _apply_band_math(dataset)
 
     #remove previous nc and write band math to disk
@@ -486,17 +501,6 @@ def create_output_products(self, data, task_id=None):
     export_xarray_to_netcdf(dataset, task.data_netcdf_path)
     write_geotiff_from_xr(task.data_path, dataset.astype('int32'), bands=bands, no_data=task.satellite.no_data_value)
 
-    # Ensure data variables have the range of Landsat 7 Collection 1 Level 2
-    # since the color scales are tailored for that dataset.
-    platform = task.satellite.platform
-    collection = task.satellite.collection
-    level = task.satellite.level
-    if (platform, collection) != ('LANDSAT_7', 'c1'):
-        dataset = \
-            convert_range(dataset, from_platform=platform, 
-                        from_collection=collection, from_level=level,
-                        to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
-
     write_png_from_xr(
         task.mosaic_path,
         dataset,

apps/tsm/tasks.py

@@ -50,16 +50,18 @@ def pixel_drill(task_id=None):
         task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
         return None
 
-    # Ensure data variables have the range of Landsat 7 Collection 1 Level 2
+    # Ensure data variables have the range of Landsat Collection 1 Level 2
     # since the color scales are tailored for that dataset.
     platform = task.satellite.platform
     collection = task.satellite.collection
     level = task.satellite.level
-    if (platform, collection) != ('LANDSAT_7', 'c1'):
+    if collection != 'c1':
+        old_dataset = single_pixel
+        drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
         single_pixel = \
-            convert_range(single_pixel, from_platform=platform, 
+            convert_range(single_pixel.drop_vars(drop_vars), from_platform=platform, 
                         from_collection=collection, from_level=level,
-                        to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
+                        to_platform=platform, to_collection='c1', to_level='l2')
 
     wofs_data = task.get_processing_method()(single_pixel,
                                              clean_mask=clear_mask,
@@ -238,7 +240,6 @@ def start_chunk_processing(self, chunk_details, task_id=None):
     # to process, also considering intermediate chunks to be combined.
     num_scenes = len(geographic_chunks) * sum([len(time_chunk) for time_chunk in time_chunks])
     # recombine_geographic_chunks() scenes:
-    # num_scn_per_chk * len(time_chunks) * len(geographic_chunks)
     num_scn_per_chk = round(num_scenes / (len(time_chunks) * len(geographic_chunks)))
     # Scene processing progress is tracked in processing_task() and recombine_geographic_chunks().
     task.total_scenes = 2 * num_scenes
@@ -326,16 +327,21 @@ def processing_task(self,
 
         clear_mask = task.satellite.get_clean_mask_func()(data)
 
-        # Ensure data variables have the range of Landsat 7 Collection 1 Level 2
+        # Ensure data variables have the range of Landsat Collection 1 Level 2
         # since the color scales are tailored for that dataset.
         platform = task.satellite.platform
         collection = task.satellite.collection
         level = task.satellite.level
-        if (platform, collection) != ('LANDSAT_7', 'c1'):
+        if collection != 'c1':
+            old_dataset = data
+            drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
             data = \
-                convert_range(data, from_platform=platform, 
+                convert_range(data.drop_vars(drop_vars), from_platform=platform, 
                             from_collection=collection, from_level=level,
-                            to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
+                            to_platform=platform, to_collection='c1', to_level='l2')
+            for drop_var in drop_vars:
+                data[drop_var] = old_dataset[drop_var]
+            data = data.transpose("time", "latitude", "longitude")
 
         wofs_data = task.get_processing_method()(data,
                                                  clean_mask=clear_mask,

apps/urbanization/tasks.py

@@ -10,7 +10,7 @@
 from utils.data_cube_utilities.data_access_api import DataAccessApi
 from utils.data_cube_utilities.dc_utilities import (create_cfmask_clean_mask, create_bit_mask, write_geotiff_from_xr,
                                                     write_png_from_xr, write_single_band_png_from_xr,
-                                                    add_timestamp_data_to_xr, clear_attrs)
+                                                    add_timestamp_data_to_xr, clear_attrs, convert_range)
 from utils.data_cube_utilities.dc_chunker import (create_geographic_chunks, create_time_chunks,
                                                   combine_geographic_chunks)
 from apps.dc_algorithm.utils import create_2d_plot, _get_datetime_range_containing
@@ -98,7 +98,6 @@ def parse_parameters_from_task(self, task_id=None):
     task = UrbanizationTask.objects.get(pk=task_id)
 
     parameters = {
-        'platform': task.satellite.datacube_platform,
         'product': task.satellite.get_products(task.area_id)[0],
         'time': (task.time_start, task.time_end),
         'longitude': (task.longitude_min, task.longitude_max),
@@ -384,10 +383,29 @@ def process_band_math(self, chunk, task_id=None):
     returns the dataarray. The data array is then appended under 'band_math', then saves the
     result to disk in the same path as the nc file already exists.
     """
+    task = UrbanizationTask.objects.get(pk=task_id)
+    if check_cancel_task(self, task): return
+
     if chunk is None:
         return None
 
     dataset = xr.open_dataset(chunk[0]).load()
+
+    # Ensure data variables have the range of Landsat Collection 1 Level 2
+    # since the color scales are tailored for that dataset.
+    platform = task.satellite.platform
+    collection = task.satellite.collection
+    level = task.satellite.level
+    if collection != 'c1':
+        old_dataset = dataset
+        drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
+        dataset = \
+            convert_range(old_dataset.drop_vars(drop_vars), from_platform=platform, 
+                          from_collection=collection, from_level=level,
+                          to_platform=platform, to_collection='c1', to_level='l2')
+        for drop_var in drop_vars:
+            dataset[drop_var] = old_dataset[drop_var]
+
     dataset['ndvi'], dataset['ndwi'], dataset['ndbi'] = _apply_band_math(dataset)
     #remove previous nc and write band math to disk
     os.remove(chunk[0])

apps/water_detection/tasks.py

@@ -47,7 +47,7 @@ def pixel_drill(task_id=None):
         task.update_status("ERROR", "There is only a single acquisition for your parameter set.")
         return None
 
-    # Ensure data variables have the range of Landsat 7 Collection 1 Level 2
+    # Ensure data variables have the range of Landsat Collection 1 Level 2
     # since the color scales are tailored for that dataset.
     platform = task.satellite.platform
     collection = task.satellite.collection
@@ -302,16 +302,21 @@ def processing_task(self,
 
         clear_mask = task.satellite.get_clean_mask_func()(data)
 
-        # Ensure data variables have the range of Landsat 7 Collection 1 Level 2
+        # Ensure data variables have the range of Landsat Collection 1 Level 2
         # since the color scales are tailored for that dataset.
         platform = task.satellite.platform
         collection = task.satellite.collection
         level = task.satellite.level
-        if (platform, collection) != ('LANDSAT_7', 'c1'):
+        if collection != 'c1':
+            old_dataset = data
+            drop_vars = [data_var for data_var in old_dataset.data_vars if data_var not in ['red', 'green', 'blue', 'nir', 'swir1', 'swir2']]
             data = \
-                convert_range(data, from_platform=platform, 
+                convert_range(old_dataset.drop_vars(drop_vars), from_platform=platform, 
                             from_collection=collection, from_level=level,
-                            to_platform='LANDSAT_7', to_collection='c1', to_level='l2')
+                            to_platform=platform, to_collection='c1', to_level='l2')
+            for drop_var in drop_vars:
+                data[drop_var] = old_dataset[drop_var]
+            data = data.transpose("time", "latitude", "longitude")
 
         wofs_data = task.get_processing_method()(data,
                                                  clean_mask=clear_mask,

build/docker/base/Dockerfile

@@ -1,8 +1,13 @@
-FROM jcrattzama/datacube-core:1.8.3
+ARG BASE_IMG
+FROM $BASE_IMG
+# jcrattzama/datacube-core:1.8.3
 
 ARG WORKDIR="/app"
 WORKDIR $WORKDIR
 
+RUN apt-get update
+RUN pip3 install setuptools
+
 # Install system dependencies.
 COPY build/native/sys_deps/install_apache_deps.sh \
      build/native/sys_deps/install_apache_deps.sh