From 51fc9a9d07e07a22db6aa3c947ab9a16940c8c58 Mon Sep 17 00:00:00 2001 From: Da7-Tech <286182457+Da7-Tech@users.noreply.github.com> Date: Wed, 20 May 2026 09:51:53 +0300 Subject: [PATCH] Add DNS first and last seen history --- dooked/include/cli_preprocessor.hpp | 6 + dooked/include/utils/exceptions.hpp | 1 + dooked/include/utils/io_utils.hpp | 18 +++ dooked/include/utils/probe_result.hpp | 3 + dooked/source/cli_preprocessor.cpp | 208 +++++++++++++++++++++++++- dooked/source/main.cpp | 7 + dooked/source/utils/io_utils.cpp | 9 ++ 7 files changed, 246 insertions(+), 6 deletions(-) diff --git a/dooked/include/cli_preprocessor.hpp b/dooked/include/cli_preprocessor.hpp index 43fa1ba..9d6bc02 100644 --- a/dooked/include/cli_preprocessor.hpp +++ b/dooked/include/cli_preprocessor.hpp @@ -25,6 +25,9 @@ struct cli_args_t { int thread_count{}; int content_length{-1}; bool include_date{false}; + bool report_first_seen{false}; + int last_seen_days{-1}; + std::string last_seen_date{}; }; struct runtime_args_t { @@ -36,6 +39,9 @@ struct runtime_args_t { http_process_e http_request_time_{}; int thread_count{}; int content_length{-1}; + bool report_first_seen{false}; + int last_seen_days{-1}; + std::string last_seen_date{}; }; void run_program(cli_args_t const &cli_args); diff --git a/dooked/include/utils/exceptions.hpp b/dooked/include/utils/exceptions.hpp index a749a1b..846d544 100644 --- a/dooked/include/utils/exceptions.hpp +++ b/dooked/include/utils/exceptions.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace dooked { diff --git a/dooked/include/utils/io_utils.hpp b/dooked/include/utils/io_utils.hpp index 829b09e..17369d9 100644 --- a/dooked/include/utils/io_utils.hpp +++ b/dooked/include/utils/io_utils.hpp @@ -26,9 +26,12 @@ void trim(std::string &); struct json_data_t { std::string domain_name{}; std::string rdata{}; + std::string first_seen{}; + std::string last_seen{}; int ttl{}; int http_code{}; int content_length{}; + int seen{}; dns_record_type_e type{}; static json_data_t serialize(std::string const &d, int const len, @@ -40,6 +43,21 @@ struct json_data_t { dns_str_to_record_type(json_object["type"].get()); data.rdata = json_object["info"].get(); data.ttl = json_object["ttl"].get(); + if (auto const first_seen_iter = json_object.find("first-seen"); + first_seen_iter != json_object.end() && + first_seen_iter->second.is_string()) { + data.first_seen = first_seen_iter->second.get(); + } + if (auto const last_seen_iter = json_object.find("last-seen"); + last_seen_iter != json_object.end() && + last_seen_iter->second.is_string()) { + data.last_seen = last_seen_iter->second.get(); + } + if (auto const seen_iter = json_object.find("seen"); + seen_iter != json_object.end() && + seen_iter->second.is_number_integer()) { + data.seen = seen_iter->second.get(); + } data.content_length = len; data.http_code = http_code; return data; diff --git a/dooked/include/utils/probe_result.hpp b/dooked/include/utils/probe_result.hpp index 07211c6..6d2adf9 100644 --- a/dooked/include/utils/probe_result.hpp +++ b/dooked/include/utils/probe_result.hpp @@ -10,8 +10,11 @@ bool case_insensitive_compare(std::string const &, std::string const &); struct probe_result_t { std::string rdata{}; + std::string first_seen{}; + std::string last_seen{}; dns_record_type_e type{}; // RR TYPE (2 octets) std::uint32_t ttl{}; // time to live(4 octets) + int seen{}; friend bool operator==(probe_result_t const &a, probe_result_t const &b) { return case_insensitive_compare(a.rdata, b.rdata) && (a.type == b.type); diff --git a/dooked/source/cli_preprocessor.cpp b/dooked/source/cli_preprocessor.cpp index c08d7fb..4bed73a 100644 --- a/dooked/source/cli_preprocessor.cpp +++ b/dooked/source/cli_preprocessor.cpp @@ -6,8 +6,11 @@ #include "utils/string_utils.hpp" #include #include +#include +#include #include #include +#include // defined (and assigned to) in main.cpp extern bool silent; @@ -18,6 +21,194 @@ namespace dooked { namespace net = boost::asio; using namespace fmt::v7::literals; +namespace { + +std::string current_history_date() { + std::string current_date{}; + if (!timet_to_string(current_date, std::time(nullptr), "%m/%d/%Y")) { + return {}; + } + return current_date; +} + +std::optional parse_history_date(std::string const &date) { + std::tm tm_date{}; + std::istringstream input{date}; + input >> std::get_time(&tm_date, "%m/%d/%Y"); + if (input.fail()) { + return std::nullopt; + } + tm_date.tm_isdst = -1; + auto parsed_time = std::mktime(&tm_date); + if (parsed_time == (std::time_t)-1) { + return std::nullopt; + } + return parsed_time; +} + +bool same_dns_record(json_data_t const &previous, std::string const &domain, + probe_result_t const ¤t) { + return case_insensitive_compare(previous.domain_name, domain) && + previous.type == current.type && + case_insensitive_compare(previous.rdata, current.rdata); +} + +std::vector::const_iterator find_previous_record( + std::vector const &previous_records, + std::string const &domain, probe_result_t const ¤t) { + return std::find_if(previous_records.cbegin(), previous_records.cend(), + [&](auto const &previous) { + return same_dns_record(previous, domain, current); + }); +} + +std::string history_first_seen(json_data_t const &previous, + std::string const &fallback_date) { + if (!previous.first_seen.empty()) { + return previous.first_seen; + } + if (!previous.last_seen.empty()) { + return previous.last_seen; + } + return fallback_date; +} + +std::string history_last_seen(json_data_t const &previous, + std::string const &fallback_date) { + if (!previous.last_seen.empty()) { + return previous.last_seen; + } + if (!previous.first_seen.empty()) { + return previous.first_seen; + } + return fallback_date; +} + +int history_seen_count(json_data_t const &previous) { + return (std::max)(1, previous.seen); +} + +bool missing_record_is_stale(json_data_t const &previous, + runtime_args_t const &rt_args, + std::time_t today_start) { + bool const days_enabled = rt_args.last_seen_days >= 0; + bool const date_enabled = !rt_args.last_seen_date.empty(); + if (!days_enabled && !date_enabled) { + return false; + } + + auto const last_seen = + parse_history_date(history_last_seen(previous, current_history_date())); + if (!last_seen) { + return days_enabled && rt_args.last_seen_days == 0; + } + + if (days_enabled) { + auto const cutoff = + today_start - static_cast(rt_args.last_seen_days) * 86400; + if (*last_seen <= cutoff) { + return true; + } + } + + if (date_enabled) { + auto const cutoff = parse_history_date(rt_args.last_seen_date); + if (!cutoff) { + spdlog::error("Invalid --lsd date `{}`; expected MM/DD/YYYY", + rt_args.last_seen_date); + return false; + } + return *last_seen <= *cutoff; + } + return false; +} + +void report_first_seen_record(std::string const &domain, + probe_result_t const &record) { + spdlog::info("[FIRST-SEEN][{}][{}] `{}`", domain, + dns_record_type_to_str(record.type), record.rdata); +} + +void report_stale_record(json_data_t const &record) { + spdlog::info("[LAST-SEEN][{}][{}][{}] `{}`", record.domain_name, + dns_record_type_to_str(record.type), + history_last_seen(record, "unknown"), record.rdata); +} + +void apply_dns_history(map_container_t &result_map, + runtime_args_t const &rt_args) { + auto const current_date = current_history_date(); + auto const *previous_records = rt_args.previous_data + ? &*rt_args.previous_data + : nullptr; + + for (auto &result_pair : result_map.result()) { + auto const &domain = result_pair.first; + for (auto ¤t_record : result_pair.second.dns_result_list_) { + if (previous_records) { + auto const previous_iter = + find_previous_record(*previous_records, domain, current_record); + if (previous_iter != previous_records->cend()) { + current_record.first_seen = + history_first_seen(*previous_iter, current_date); + current_record.last_seen = current_date; + current_record.seen = history_seen_count(*previous_iter) + 1; + continue; + } + } + + current_record.first_seen = current_date; + current_record.last_seen = current_date; + current_record.seen = 1; + if (rt_args.report_first_seen) { + report_first_seen_record(domain, current_record); + } + } + } + + if (!previous_records) { + return; + } + + auto const today = parse_history_date(current_date); + auto const today_start = today.value_or(std::time(nullptr)); + + for (auto const &previous_record : *previous_records) { + auto const current_domain_iter = + result_map.result().find(previous_record.domain_name); + bool const found = + current_domain_iter != result_map.result().end() && + std::find_if(current_domain_iter->second.dns_result_list_.cbegin(), + current_domain_iter->second.dns_result_list_.cend(), + [&](auto const ¤t_record) { + return same_dns_record(previous_record, + previous_record.domain_name, + current_record); + }) != current_domain_iter->second.dns_result_list_.cend(); + + if (found) { + continue; + } + + if (missing_record_is_stale(previous_record, rt_args, today_start)) { + report_stale_record(previous_record); + } + + probe_result_t preserved_record{}; + preserved_record.rdata = previous_record.rdata; + preserved_record.first_seen = + history_first_seen(previous_record, current_date); + preserved_record.last_seen = + history_last_seen(previous_record, current_date); + preserved_record.type = previous_record.type; + preserved_record.ttl = previous_record.ttl; + preserved_record.seen = history_seen_count(previous_record); + result_map.append(previous_record.domain_name, preserved_record); + } +} + +} // namespace + void compare_http_result(int const base_cl, json_data_t const &prev_http_result, http_response_t const ¤t_result) { auto const current_req_cl = current_result.content_length_; @@ -351,11 +542,8 @@ void start_name_checking(runtime_args_t &&rt_args) { thread_pool->join(); } if (!silent) { - spdlog::info("Writing JSON output"); + spdlog::info("Preparing DNS history"); } - write_json_result(result_map, rt_args); - - // compare old with new result -- only if we had previous record if (rt_args.previous_data) { auto &previous_data = *rt_args.previous_data; @@ -373,9 +561,14 @@ void start_name_checking(runtime_args_t &&rt_args) { return std::tie(a.type, a.rdata) < std::tie(b.type, b.rdata); }); } - return compare_results(*rt_args.previous_data, result_map, - rt_args.content_length); + compare_results(*rt_args.previous_data, result_map, + rt_args.content_length); } + apply_dns_history(result_map, rt_args); + if (!silent) { + spdlog::info("Writing JSON output"); + } + write_json_result(result_map, rt_args); } void run_program(cli_args_t const &cli_args) { @@ -477,6 +670,9 @@ void run_program(cli_args_t const &cli_args) { static_cast(cli_args.post_http_request); rt_args.thread_count = cli_args.thread_count; rt_args.content_length = cli_args.content_length; + rt_args.report_first_seen = cli_args.report_first_seen; + rt_args.last_seen_days = cli_args.last_seen_days; + rt_args.last_seen_date = cli_args.last_seen_date; return start_name_checking(std::move(rt_args)); } diff --git a/dooked/source/main.cpp b/dooked/source/main.cpp index cf29460..a264ba9 100644 --- a/dooked/source/main.cpp +++ b/dooked/source/main.cpp @@ -41,6 +41,13 @@ int main(int argc, char **argv) { "defers http request until after all DNS requests have been completed"); app.add_flag("--compare-cl", compare_cl, "compare content-length of HTTP requests"); + app.add_flag("--fs", cli_args.report_first_seen, + "show records first seen in the current run"); + app.add_option("--ls", cli_args.last_seen_days, + "show records missing from this run and not seen in N days"); + app.add_option("--lsd", cli_args.last_seen_date, + "show records missing from this run and not seen since " + "MM/DD/YYYY"); app.add_flag("--nbc", no_bytes_count, "in case `content-length` is missing in an HTTP header field," diff --git a/dooked/source/utils/io_utils.cpp b/dooked/source/utils/io_utils.cpp index a1bd5d3..01541d5 100644 --- a/dooked/source/utils/io_utils.cpp +++ b/dooked/source/utils/io_utils.cpp @@ -6,6 +6,15 @@ void to_json(json &j, probe_result_t const &record) { j = json{{"ttl", record.ttl}, {"type", dns_record_type_to_str(record.type)}, {"info", record.rdata}}; + if (!record.first_seen.empty()) { + j["first-seen"] = record.first_seen; + } + if (!record.last_seen.empty()) { + j["last-seen"] = record.last_seen; + } + if (record.seen > 0) { + j["seen"] = record.seen; + } } bool is_text_file(std::string const &file_extension) {