Spike submissions stats

app/controllers/reports_controller.rb

@@ -8,8 +8,9 @@ def features
     tag = params[:tag]
     forms = Reports::FormDocumentsService.form_documents(tag:)
     data = Reports::FeatureReportService.new(forms).report
+    total_submissions = total_submissions_for_features(tag)
 
-    render template: "reports/features", locals: { tag:, data: }
+    render template: "reports/features", locals: { tag:, data:, total_submissions: }
   end
 
   def questions_with_answer_type
@@ -202,6 +203,11 @@ def contact_for_research
     render locals: { data: }
   end
 
+  def total_submissions
+    data = Reports::TotalSubmissionsCloudWatchService.new.submissions_data
+    render locals: { data: }
+  end
+
 private
 
   def questions_feature_report(tag, report, questions, type: :questions)
@@ -224,6 +230,16 @@ def forms_feature_report(tag, report, forms, type: :forms)
     end
   end
 
+  def total_submissions_for_features(tag)
+    return nil unless tag == "live-or-archived"
+
+    submissions_data = Reports::TotalSubmissionsCloudWatchService.new.submissions_data
+    {
+      total: submissions_data&.dig(:all_time, :total),
+      available: !submissions_data.nil?,
+    }
+  end
+
   def check_user_has_permission
     authorize :report, :can_view_reports?
   end

app/services/reports/total_submissions_cloud_watch_service.rb

@@ -0,0 +1,134 @@
+class Reports::TotalSubmissionsCloudWatchService
+  REGION = "eu-west-2".freeze
+
+  def submissions_data
+    return nil unless Settings.cloudwatch_metrics_enabled
+    return nil if Settings.total_submissions_baseline_cutoff_date.blank?
+
+    datapoints = fetch_daily_datapoints
+    cloudwatch_total = datapoints.values.sum
+    baseline = Settings.total_submissions_baseline.to_i
+
+    {
+      all_time: { total: baseline + cloudwatch_total },
+      year: {
+        in_progress: year_in_progress_bucket(datapoints),
+      },
+      month: month_buckets(datapoints),
+      week: week_buckets(datapoints),
+      day: day_buckets(datapoints),
+      weekly_breakdown: weekly_breakdown(datapoints),
+      monthly_breakdown: monthly_breakdown(datapoints),
+    }
+  rescue Aws::CloudWatch::Errors::ServiceError,
+         Aws::Errors::MissingCredentialsError,
+         ArgumentError => e
+    Sentry.capture_exception(e)
+    nil
+  end
+
+private
+
+  def fetch_daily_datapoints
+    env = Settings.forms_env.downcase
+    expression = "SUM(SEARCH('{Forms,Environment,FormId} MetricName=\"Submitted\" Environment=\"#{env}\"', 'Sum', 86400))"
+
+    start_time = Date.iso8601(Settings.total_submissions_baseline_cutoff_date).beginning_of_day.utc
+
+    response = Aws::CloudWatch::Client.new(region: REGION).get_metric_data({
+      metric_data_queries: [
+        {
+          id: "total_submissions",
+          expression:,
+          label: "Total Submissions",
+        },
+      ],
+      start_time: start_time,
+      end_time: Time.zone.now.utc,
+    })
+
+    result = response.metric_data_results.find { |r| r.id == "total_submissions" }
+    return {} unless result
+
+    result.timestamps.zip(result.values).each_with_object({}) do |(timestamp, value), hash|
+      hash[timestamp.to_date.iso8601] = value.to_i
+    end
+  end
+
+  def sum_dates(datapoints, start_date, end_date)
+    (start_date..end_date).sum { |date| datapoints[date.iso8601] || 0 }
+  end
+
+  def today
+    Time.zone.today
+  end
+
+  def day_buckets(datapoints)
+    yesterday = today - 1.day
+    {
+      completed: { label: yesterday.strftime("%-d %b %Y"), total: datapoints[yesterday.iso8601] || 0 },
+      in_progress: { label: today.strftime("%-d %b %Y"), total: datapoints[today.iso8601] || 0 },
+    }
+  end
+
+  def week_buckets(datapoints)
+    this_week_start = today.beginning_of_week(:monday)
+    last_week_end   = this_week_start - 1.day
+    last_week_start = last_week_end.beginning_of_week(:monday)
+
+    {
+      completed: { label: week_label(last_week_start, last_week_end), total: sum_dates(datapoints, last_week_start, last_week_end) },
+      in_progress: { label: week_label(this_week_start, today), total: sum_dates(datapoints, this_week_start, today) },
+    }
+  end
+
+  def month_buckets(datapoints)
+    this_month_start = today.beginning_of_month
+    last_month_end   = this_month_start - 1.day
+    last_month_start = last_month_end.beginning_of_month
+
+    {
+      completed: { label: last_month_start.strftime("%B %Y"), total: sum_dates(datapoints, last_month_start, last_month_end) },
+      in_progress: { label: this_month_start.strftime("%B %Y"), total: sum_dates(datapoints, this_month_start, today) },
+    }
+  end
+
+  def year_in_progress_bucket(datapoints)
+    this_year_start = Date.new(today.year, 1, 1)
+
+    {
+      label: this_year_start.year.to_s,
+      total: sum_dates(datapoints, this_year_start, today),
+    }
+  end
+
+  def weekly_breakdown(datapoints)
+    this_week_start = today.beginning_of_week(:monday)
+    last_week_end   = this_week_start - 1.day
+
+    (0...52).map do |i|
+      week_end   = last_week_end - (i * 7)
+      week_start = week_end - 6.days
+      { label: week_label(week_start, week_end), total: sum_dates(datapoints, week_start, week_end) }
+    end
+  end
+
+  def monthly_breakdown(datapoints)
+    this_month_start = today.beginning_of_month
+    (1..12).map do |i|
+      month_start = this_month_start - i.months
+      month_end   = month_start.end_of_month
+      { label: month_start.strftime("%B %Y"), total: sum_dates(datapoints, month_start, month_end) }
+    end
+  end
+
+  def week_label(start_date, end_date)
+    if start_date.year != end_date.year
+      "#{start_date.strftime('%-d %b %Y')}–#{end_date.strftime('%-d %b %Y')}"
+    elsif start_date.month != end_date.month
+      "#{start_date.strftime('%-d %b')}–#{end_date.strftime('%-d %b %Y')}"
+    else
+      "#{start_date.day}–#{end_date.strftime('%-d %b %Y')}"
+    end
+  end
+end

app/views/reports/features.html.erb

@@ -10,6 +10,12 @@
         <%= row.with_key(text: t(".features.total_forms", tag: tag_label(tag)).upcase_first) %>
         <%= row.with_value(text: data[:total_forms]) %>
       <% end %>
+      <% if total_submissions %>
+        <%= summary_list.with_row do |row| %>
+          <%= row.with_key(text: t(".features.total_submissions").upcase_first) %>
+          <%= row.with_value(text: total_submissions[:available] ? total_submissions[:total] : t(".features.total_submissions_unavailable")) %>
+        <% end %>
+      <% end %>
       <%= summary_list.with_row do |row| %>
         <%= row.with_key(text: t(".features.copied_forms", tag: tag_label(tag)).upcase_first) %>
         <%= row.with_value(text: govuk_link_to(data[:copied_forms], report_forms_that_are_copies_path, no_visited_state: true)) %>

app/views/reports/index.html.erb

@@ -12,6 +12,7 @@
       <li><%= govuk_link_to t("reports.last_signed_in_at.title"), report_last_signed_in_at_path %></li>
       <li><%= govuk_link_to t("reports.csv_downloads.title"), report_csv_downloads_path %></li>
       <li><%= govuk_link_to t("reports.contact_for_research.title"), report_contact_for_research_path %></li>
+      <li><%= govuk_link_to t("reports.total_submissions.title"), report_total_submissions_path %></li>
     </ul>
   </div>
 </div>

app/views/reports/total_submissions.html.erb

@@ -0,0 +1,86 @@
+<% set_page_title(t(".title")) %>
+<% content_for :back_link, govuk_back_link_to(reports_path, t("reports.back_link")) %>
+<div class="govuk-grid-row">
+  <div class="govuk-grid-column-two-thirds">
+    <h1 class="govuk-heading-l"><%= t(".title") %></h1>
+
+    <% if data.nil? %>
+      <div class="govuk-inset-text">
+        <%= t(".error_loading_data") %>
+      </div>
+    <% else %>
+      <%= govuk_table do |table| %>
+        <%= table.with_caption(size: "m", text: t(".summary.heading")) %>
+        <%= table.with_head do |head| %>
+          <%= head.with_row do |row| %>
+            <%= row.with_cell(text: t(".summary.period")) %>
+            <%= row.with_cell(text: t(".summary.count"), numeric: true) %>
+          <% end %>
+        <% end %>
+        <%= table.with_body do |body| %>
+          <% [:day, :year].each do |period| %>
+            <%= body.with_row do |row| %>
+              <%= row.with_cell(header: true, text: "#{data[period][:in_progress][:label]} #{t(".summary.in_progress_suffix")}") %>
+              <%= row.with_cell(text: data[period][:in_progress][:total], numeric: true) %>
+            <% end %>
+            <% if data[period][:completed].present? %>
+              <%= body.with_row do |row| %>
+                <%= row.with_cell(header: true, text: data[period][:completed][:label]) %>
+                <%= row.with_cell(text: data[period][:completed][:total], numeric: true) %>
+              <% end %>
+            <% end %>
+          <% end %>
+          <%= body.with_row do |row| %>
+            <%= row.with_cell(header: true, text: t(".summary.all_time")) %>
+            <%= row.with_cell(text: data[:all_time][:total], numeric: true) %>
+          <% end %>
+        <% end %>
+      <% end %>
+
+      <%= govuk_table do |table| %>
+        <%= table.with_caption(size: "m", text: t(".weekly_breakdown.heading")) %>
+        <%= table.with_head do |head| %>
+          <%= head.with_row do |row| %>
+            <%= row.with_cell(text: t(".weekly_breakdown.week")) %>
+            <%= row.with_cell(text: t(".weekly_breakdown.count"), numeric: true) %>
+          <% end %>
+        <% end %>
+        <%= table.with_body do |body| %>
+          <%= body.with_row do |row| %>
+            <%= row.with_cell(header: true, text: "#{data[:week][:in_progress][:label]} #{t(".summary.in_progress_suffix")}") %>
+            <%= row.with_cell(text: data[:week][:in_progress][:total], numeric: true) %>
+          <% end %>
+
+          <% data[:weekly_breakdown].each do |week| %>
+            <%= body.with_row do |row| %>
+              <%= row.with_cell(text: week[:label]) %>
+              <%= row.with_cell(text: week[:total], numeric: true) %>
+            <% end %>
+          <% end %>
+        <% end %>
+      <% end %>
+
+      <%= govuk_table do |table| %>
+        <%= table.with_caption(size: "m", text: t(".monthly_breakdown.heading")) %>
+        <%= table.with_head do |head| %>
+          <%= head.with_row do |row| %>
+            <%= row.with_cell(text: t(".monthly_breakdown.month")) %>
+            <%= row.with_cell(text: t(".monthly_breakdown.count"), numeric: true) %>
+          <% end %>
+        <% end %>
+        <%= table.with_body do |body| %>
+          <%= body.with_row do |row| %>
+            <%= row.with_cell(header: true, text: "#{data[:month][:in_progress][:label]} #{t(".summary.in_progress_suffix")}") %>
+            <%= row.with_cell(text: data[:month][:in_progress][:total], numeric: true) %>
+          <% end %>
+          <% data[:monthly_breakdown].each do |month| %>
+            <%= body.with_row do |row| %>
+              <%= row.with_cell(text: month[:label]) %>
+              <%= row.with_cell(text: month[:total], numeric: true) %>
+            <% end %>
+          <% end %>
+        <% end %>
+      <% end %>
+    <% end %>
+  </div>
+</div>

config/locales/en.yml

@@ -1836,6 +1836,8 @@ en:
         forms_with_welsh_translation: "%{tag} forms with Welsh translation"
         heading: Feature usage
         total_forms: Total %{tag} forms
+        total_submissions: Total submissions (all time)
+        total_submissions_unavailable: Data unavailable
       title: Feature and answer type usage in %{tag} forms
     form_or_questions_list_table:
       headings:
@@ -1958,6 +1960,23 @@ en:
       draft: draft
       live: live
       live-or-archived: live or archived
+    total_submissions:
+      error_loading_data: Sorry, the total submissions data could not be loaded. Please try again later.
+      monthly_breakdown:
+        count: Total submissions
+        heading: Monthly breakdown
+        month: Month
+      summary:
+        all_time: All time
+        count: Total submissions
+        heading: Submission totals
+        in_progress_suffix: "(so far)"
+        period: Period
+      title: Total form submissions
+      weekly_breakdown:
+        count: Total submissions
+        heading: Weekly breakdown
+        week: Week
     users:
       heading: Number of users per organisation
       table_headings:

config/routes.rb

@@ -279,6 +279,7 @@
     get "live-forms-csv", to: "reports#live_forms_csv", as: :report_live_forms_csv
     get "live-questions-csv", to: "reports#live_questions_csv", as: :report_live_questions_csv
     get "contact-for-research", to: "reports#contact_for_research", as: :report_contact_for_research
+    get "total-submissions", to: "reports#total_submissions", as: :report_total_submissions
   end
 
   scope "api/v2", as: "api_v2" do

config/settings.yml

@@ -109,5 +109,9 @@ act_as_user_enabled: false
 
 cloudwatch_metrics_enabled: true
 
+# Total form submissions recorded before the cutoff date.
+total_submissions_baseline: 0
+total_submissions_baseline_cutoff_date: "2025-02-01"
+
 reports:
   forms_api_forms_per_request_page: 100

config/settings/test.yml

@@ -25,6 +25,8 @@ mailchimp:
   mou_signers_list: list-2
 
 cloudwatch_metrics_enabled: false
+total_submissions_baseline: 0
+total_submissions_baseline_cutoff_date: "2025-02-01"
 
 reports:
   forms_api_forms_per_request_page: 3