Skip to content

Commit e7d8032

Browse files
marcoSantiMarco Edoardo Santimaria
marcoSanti
and
Marco Edoardo Santimaria
committed
Added Commit-N-Files rule parsing within the CAPIO-CL json parser (#153)
Co-authored-by: Marco Edoardo Santimaria <marcoedoardo.santimaria@unito.it>
1 parent 52dac6f commit e7d8032

3 files changed

Lines changed: 48 additions & 27 deletions

File tree

‎src/common/capio/constants.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ constexpr char CAPIO_FILE_MODE_NO_UPDATE[] = "no_update";
5151
constexpr char CAPIO_FILE_MODE_UPDATE[] = "update";
5252
constexpr char CAPIO_FILE_COMMITTED_ON_CLOSE[] = "on_close";
5353
constexpr char CAPIO_FILE_COMMITTED_ON_FILE[] = "on_file";
54+
constexpr char CAPIO_FILE_COMMITTED_N_FILES[] = "n_files";
5455
constexpr char CAPIO_FILE_COMMITTED_ON_TERMINATION[] = "on_termination";
5556

5657
// CAPIO POSIX return codes

‎src/server/capio-cl-engine/capio_cl_engine.hpp‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,29 +38,30 @@ class CapioCLEngine {
3838
<< "Composition of expected CAPIO FS: " << std::endl
3939
<< std::endl
4040
<< "|============================================================================"
41-
"===============================================|"
41+
"==========================================================|"
4242
<< std::endl
43-
<< "|" << std::setw(124) << "|" << std::endl
43+
<< "|" << std::setw(135) << "|" << std::endl
4444
<< "| Parsed configuration file for workflow: \e[1;36m" << workflow_name
45-
<< std::setw(83 - workflow_name.length()) << "\e[0m |" << std::endl
46-
<< "|" << std::setw(124) << "|" << std::endl
45+
<< std::setw(94 - workflow_name.length()) << "\e[0m |" << std::endl
46+
<< "|" << std::setw(135) << "|" << std::endl
4747
<< "| File color legend: \e[48;5;034m \e[0m File stored in memory"
48-
<< std::setw(72) << "|" << std::endl
48+
<< std::setw(83) << "|" << std::endl
4949
<< "| "
50-
<< "\e[48;5;172m \e[0m File stored on file system" << std::setw(67) << "|"
50+
<< "\e[48;5;172m \e[0m File stored on file system" << std::setw(78) << "|"
5151
<< std::endl
5252
<< "|============================================================================"
53-
"===============================================|"
53+
"==========================================================|"
5454
<< std::endl
5555
<< "|======|===================|===================|====================|========"
56-
"============|============|===========|=========|"
56+
"============|============|===========|=========|==========|"
5757
<< std::endl
5858
<< "| Kind | Filename | Producer step | Consumer step | "
59-
"Commit Rule | Fire Rule | Permanent | Exclude |"
59+
"Commit Rule | Fire Rule | Permanent | Exclude | n_files |"
6060
<< std::endl
6161
<< "|======|===================|===================|====================|========"
62-
"============|============|===========|=========|"
62+
"============|============|===========|=========|==========|"
6363
<< std::endl;
64+
6465
for (auto itm : _locations) {
6566
std::string color_preamble = std::get<11>(itm.second) ? "\e[38;5;034m" : "\e[38;5;172m";
6667
std::string color_post = "\e[0m";
@@ -76,6 +77,12 @@ class CapioCLEngine {
7677
auto rowCount =
7778
producers.size() > consumers.size() ? producers.size() : consumers.size();
7879

80+
// Add logic to handle the n_files column
81+
std::string n_files = std::to_string(std::get<8>(itm.second));
82+
if (std::get<8>(itm.second) < 1) {
83+
n_files = "N.A.";
84+
}
85+
7986
for (int i = 0; i <= rowCount; i++) {
8087
std::string prod, cons;
8188
if (i > 0) {
@@ -107,15 +114,18 @@ class CapioCLEngine {
107114
<< std::setw(20 - commit_rule.length()) << " | " << fire_rule
108115
<< std::setfill(' ') << std::setw(13 - fire_rule.length()) << " | "
109116
<< " " << (permanent ? "YES" : "NO ") << " | "
110-
<< (exclude ? "YES" : "NO ") << " |" << std::endl;
117+
<< (exclude ? "YES" : "NO ") << " | " << n_files
118+
<< std::setw(11 - n_files.length()) << " | "
119+
<< std::endl;
111120
} else {
112121
std::cout << std::setfill(' ') << std::setw(20) << "|" << std::setfill(' ')
113122
<< std::setw(13) << "|" << std::setfill(' ') << std::setw(12) << "|"
114-
<< std::setfill(' ') << std::setw(10) << "|" << std::endl;
123+
<< std::setfill(' ') << std::setw(10) << "|" << std::setw(11) << "|"
124+
<< std::endl;
115125
}
116126
}
117127
std::cout << "*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
118-
"~~~~~~~"
128+
"~~~~~~~~~~~~~~~~~~"
119129
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*"
120130
<< std::endl;
121131
}

‎src/server/capio-cl-engine/json_parser.hpp‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#ifndef JSON_PARSER_HPP
22
#define JSON_PARSER_HPP
3+
#include "capio/constants.hpp"
4+
35
#include <singleheader/simdjson.h>
46

57
/**
@@ -184,8 +186,8 @@ class JsonParser {
184186
std::vector<std::filesystem::path> streaming_names;
185187
std::vector<std::string> file_deps;
186188
long int n_close = -1;
187-
long n_files, batch_size;
188-
bool is_file = true;
189+
long n_files = -1, batch_size;
190+
bool is_file = true;
189191

190192
simdjson::ondemand::array name;
191193
error = file["name"].get_array().get(name);
@@ -228,22 +230,28 @@ class JsonParser {
228230
auto pos = committed.find(':');
229231
if (pos != std::string::npos) {
230232
commit_rule = committed.substr(0, pos);
231-
if (commit_rule != CAPIO_FILE_COMMITTED_ON_CLOSE) {
233+
std::string count_str(committed.substr(pos + 1, committed.length()));
234+
if (!is_int(count_str)) {
232235
std::cout << CAPIO_LOG_SERVER_CLI_LEVEL_ERROR << " [ " << node_name
233236
<< " ] "
234-
<< "commit rule " << commit_rule << std::endl;
235-
ERR_EXIT("error commit rule: %s", std::string(commit_rule).c_str());
237+
<< "commit rule on_close/n_files invalid number"
238+
<< std::endl;
239+
ERR_EXIT("error commit rule on_close invalid number: !is_int()");
236240
}
237241

238-
std::string n_close_str(committed.substr(pos + 1, committed.length()));
239-
240-
if (!is_int(n_close_str)) {
242+
if (commit_rule == CAPIO_FILE_COMMITTED_ON_CLOSE) {
243+
n_close = std::stol(count_str);
244+
} else if (commit_rule == CAPIO_FILE_COMMITTED_N_FILES) {
245+
n_files = std::stol(count_str);
246+
// TODO: use internally n_files. for now, we use on_close as default
247+
commit_rule = CAPIO_FILE_COMMITTED_ON_CLOSE;
248+
} else {
241249
std::cout << CAPIO_LOG_SERVER_CLI_LEVEL_ERROR << " [ " << node_name
242250
<< " ] "
243-
<< "commit rule on_close invalid number" << std::endl;
244-
ERR_EXIT("error commit rule on_close invalid number: !is_int()");
251+
<< "commit rule " << commit_rule << std::endl;
252+
ERR_EXIT("error commit rule: %s", std::string(commit_rule).c_str());
245253
}
246-
n_close = std::stol(n_close_str);
254+
247255
} else {
248256
commit_rule = committed;
249257
}
@@ -292,9 +300,11 @@ class JsonParser {
292300
}
293301
LOG("Mode: %s", std::string(mode).c_str());
294302

295-
error = file["n_files"].get_int64().get(n_files);
296-
if (error) {
297-
n_files = -1;
303+
if (n_files == -1) {
304+
error = file["n_files"].get_int64().get(n_files);
305+
if (error && n_files != -1) {
306+
n_files = -1;
307+
}
298308
}
299309
LOG("n_files: %d", n_files);
300310

0 commit comments

Comments
 (0)