Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions components/omega/OmegaBuild.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -574,11 +574,6 @@ macro(update_variables)
add_definitions(-DOMEGA_LOG_FLUSH)
endif()

if(OMEGA_LOG_TASKS)
string(TOUPPER "${OMEGA_LOG_TASKS}" _LOG_TASKS)
add_definitions(-DOMEGA_LOG_TASKS=${_LOG_TASKS})
endif()

if(OMEGA_MEMORY_LAYOUT)
string(TOUPPER "${OMEGA_MEMORY_LAYOUT}" _LAYOUT)
add_definitions(-DOMEGA_LAYOUT_${_LAYOUT})
Expand Down
18 changes: 12 additions & 6 deletions components/omega/doc/design/Logging.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,18 @@ compile time, where `<pattern>` represents a format in the form of `%flag`,
similar to the strftime function. This pattern defines the layout of the log
message.

Users can control which MPI ranks generate log files using
`-D OMEGA_LOG_TASKS=<tasks-pattern>` at compile time. The `<tasks-pattern>` is
either all for all tasks to generate log files, or comma-separated MPI rank
numbers, or a range of MPI ranks with a dash. For example,
`-D OMEGA_LOG_TASKS=0,2-3` indicates that MPI ranks 0, 2, and 3 generate log
files.
Users can control which MPI ranks generate log files using the
`OMEGA_LOG_TASKS` selector, which is resolved at runtime against the Omega MPI
sub-communicator. The selector is read from the `OMEGA_LOG_TASKS` environment
variable; when it is unset, logging defaults to the master rank only. The
`<selector>` accepts `*` (all ranks), `m` or `master` (the sub-communicator
master rank), a single rank number, a comma-separated list of ranks, an
inclusive dash range, or any combination of lists and ranges. For example,
`OMEGA_LOG_TASKS=0,2-3` makes MPI ranks 0, 2, and 3 generate log files. A
malformed selector logs a warning on the master rank and falls back to
master-rank-only logging; ranks outside the sub-communicator simply produce no
log. When setting the `*` selector via the environment variable, quote it (for
example `OMEGA_LOG_TASKS='*'`) to prevent shell glob expansion.

#### 4.1.2 Class/structs/data types

Expand Down
1 change: 0 additions & 1 deletion components/omega/doc/devGuide/CMakeBuild.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ OMEGA_MEMORY_LAYOUT: Kokkos memory layout ("LEFT" or "RIGHT"). "RIGHT" is a defa
OMEGA_TILE_LENGTH: a length of one "side" of a Kokkos tile. 64 is a default value.
OMEGA_LOG_LEVEL: a default logging level. "OMEGA_LOG_INFO" is a default value.
OMEGA_LOG_FLUSH: turn on the unbuffered logging. "OFF" is a default value.
OMEGA_LOG_TASKS: set the tasks that generate log file. "0" is a default value.
OMEGA_VECTOR_LENGTH: Vector length used for blocking inner loops for vectorization. "1" is a default value.
```

Expand Down
206 changes: 152 additions & 54 deletions components/omega/src/infra/Logging.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,17 @@
//===----------------------------------------------------------------------===//
#include "Logging.h"
#include "MachEnv.h"
#include <algorithm>
#include <cctype>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <limits>
#include <set>
#include <spdlog/sinks/basic_file_sink.h>

#define _OMEGA_STRINGIFY(x) #x
#define _OMEGA_TOSTRING(x) _OMEGA_STRINGIFY(x)
#include <sstream>
#include <vector>

namespace OMEGA {

Expand All @@ -39,52 +43,144 @@ _PackLogMsg(const char *file, // [in] file from where log called (cpp __FILE__)
}

//------------------------------------------------------------------------------
// Utility function to determine which tasks perform logging based on
// input variable string containing lists of tasks or other options
std::vector<int>
splitTasks(const std::string &str, //[in] option for which tasks should write
const OMEGA::I4 NumTasks //[in] total number of tasks
) {
// Trim surrounding whitespace from a string
static std::string trimStr(const std::string &Str) {
const std::string WhiteSpace = " \t\n\r\f\v";
std::size_t Begin = Str.find_first_not_of(WhiteSpace);
if (Begin == std::string::npos)
return "";
std::size_t End = Str.find_last_not_of(WhiteSpace);
return Str.substr(Begin, End - Begin + 1);
}

std::vector<int> Tasks;
std::stringstream ss(str);
std::string Task;
int Start, Stop;
char Dash;
//------------------------------------------------------------------------------
// Parse a string as a non-negative int. Returns false if the string is empty,
// contains any non-digit character, or overflows an int.
static bool toNonNegInt(const std::string &Str, int &Out) {
if (Str.empty())
return false;
for (char C : Str) {
if (!std::isdigit(static_cast<unsigned char>(C)))
return false;
}
try {
std::size_t Pos = 0;
long Val = std::stol(Str, &Pos);
if (Pos != Str.size() || Val < 0 ||
Val > static_cast<long>(std::numeric_limits<int>::max()))
return false;
Out = static_cast<int>(Val);
return true;
} catch (...) {
return false;
}
}

// If all tasks should write, create the explicit list of all tasks
if (str == "ALL") {
for (int i = 0; i < NumTasks; ++i) {
Tasks.push_back(i);
}
return Tasks;
//------------------------------------------------------------------------------
// Resolve a logging task selector string into the list of ranks that should log
std::vector<int> _selectLogTasks(const std::string &Selector,
OMEGA::I4 NumTasks, OMEGA::I4 MasterTask,
bool &Valid) {

Valid = true;

std::string Sel = trimStr(Selector);

// Lowercase copy for case-insensitive keyword matching
std::string Lower = Sel;
std::transform(Lower.begin(), Lower.end(), Lower.begin(),
[](unsigned char C) { return std::tolower(C); });

// Standalone keyword: all ranks in the sub-communicator
if (Lower == "*") {
std::vector<int> All;
for (int I = 0; I < NumTasks; ++I)
All.push_back(I);
return All;
}

// Parse the task string. If there is a single task, extract the task id.
// If there is a range (beg:end), create a list with that range
while (std::getline(ss, Task, ':')) {
std::istringstream iss(Task);
iss >> Start;
// Standalone keyword: master rank only
if (Lower == "m" || Lower == "master")
return std::vector<int>{MasterTask};

if (iss.eof()) { // no range found, so extract single task
Tasks.push_back(Start);
// Empty selector is malformed
if (Sel.empty()) {
Valid = false;
return std::vector<int>{MasterTask};
}

// Numeric expression: comma-separated single ranks and/or "a-b" ranges.
// A std::set keeps the result sorted and deduplicated.
std::set<int> Selected;
std::stringstream Ss(Sel);
std::string Token;
while (std::getline(Ss, Token, ',')) {
Token = trimStr(Token);

std::size_t DashPos = Token.find('-');
if (DashPos == std::string::npos) {
int Rank;
if (!toNonNegInt(Token, Rank)) {
Valid = false;
return std::vector<int>{MasterTask};
}
Selected.insert(Rank);
} else {
iss >> Dash;
iss >> Stop;

for (int i = Start; i <= Stop; ++i) {
Tasks.push_back(i);
int Lo, Hi;
if (!toNonNegInt(trimStr(Token.substr(0, DashPos)), Lo) ||
!toNonNegInt(trimStr(Token.substr(DashPos + 1)), Hi) || Lo > Hi) {
Valid = false;
return std::vector<int>{MasterTask};
}
for (int I = Lo; I <= Hi; ++I)
Selected.insert(I);
}
}

// Default to all tasks if a -1 task found in list
if (std::find(Tasks.begin(), Tasks.end(), -1) != Tasks.end()) {
Tasks.clear();
for (int i = 0; i < NumTasks; ++i) {
Tasks.push_back(i);
}
return std::vector<int>(Selected.begin(), Selected.end());
}

//------------------------------------------------------------------------------
// Source the logging task selector from the OMEGA_LOG_TASKS environment
// variable, falling back to the master rank when unset/empty.
static std::string getLogTaskSelector() {
const char *Env = std::getenv("OMEGA_LOG_TASKS");
if (Env != nullptr && Env[0] != '\0')
return std::string(Env);
return std::string("master");
}

//------------------------------------------------------------------------------
// Resolve which sub-communicator ranks should log from the runtime selector,
// emitting master-rank diagnostics for invalid or empty selections. Sets
// NumLogging to the count of selected ranks that exist in this communicator.
static std::vector<int> resolveLogTasks(const OMEGA::MachEnv *DefEnv,
int &NumLogging) {

OMEGA::I4 NumTasks = DefEnv->getNumTasks();
OMEGA::I4 MasterTask = DefEnv->getMasterTask();

std::string Selector = getLogTaskSelector();
bool Valid = false;
std::vector<int> Tasks =
_selectLogTasks(Selector, NumTasks, MasterTask, Valid);

if (!Valid && DefEnv->isMasterTask()) {
std::cerr << "[Omega Logging] Invalid OMEGA_LOG_TASKS selector \""
<< Selector << "\"; falling back to master rank only."
<< std::endl;
}

// Count selected ranks that actually exist in this communicator
NumLogging = 0;
for (int Rank : Tasks) {
if (Rank >= 0 && Rank < NumTasks)
++NumLogging;
}
if (Valid && NumLogging == 0 && DefEnv->isMasterTask()) {
std::cerr << "[Omega Logging] OMEGA_LOG_TASKS selector \"" << Selector
<< "\" matches no ranks in this communicator; logging disabled."
<< std::endl;
}

return Tasks;
Expand All @@ -100,15 +196,15 @@ int initLogging(

int RetVal = 0;

OMEGA::I4 TaskId = DefEnv->getMyTask();
OMEGA::I4 NumTasks = DefEnv->getNumTasks();
OMEGA::I4 TaskId = DefEnv->getMyTask();

// determine which tasks will write log files
std::vector<int> Tasks =
splitTasks(_OMEGA_TOSTRING(OMEGA_LOG_TASKS), NumTasks);
int NumLogging;
std::vector<int> Tasks = resolveLogTasks(DefEnv, NumLogging);

bool ThisTaskLogs =
(std::find(Tasks.begin(), Tasks.end(), TaskId) != Tasks.end());

if (Tasks.size() > 0 &&
(std::find(Tasks.begin(), Tasks.end(), TaskId) != Tasks.end())) {
if (ThisTaskLogs) {

spdlog::set_default_logger(Logger);

Expand Down Expand Up @@ -140,22 +236,24 @@ int initLogging(

int RetVal = 0;

OMEGA::I4 TaskId = DefEnv->getMyTask();
OMEGA::I4 NumTasks = DefEnv->getNumTasks();
OMEGA::I4 TaskId = DefEnv->getMyTask();

std::string NewLogFilePath;

// Determine which tasks will write log files
std::vector<int> Tasks =
splitTasks(_OMEGA_TOSTRING(OMEGA_LOG_TASKS), NumTasks);
int NumLogging;
std::vector<int> Tasks = resolveLogTasks(DefEnv, NumLogging);

if (Tasks.size() > 0 &&
(std::find(Tasks.begin(), Tasks.end(), TaskId) != Tasks.end())) {
bool ThisTaskLogs =
(std::find(Tasks.begin(), Tasks.end(), TaskId) != Tasks.end());

if (ThisTaskLogs) {
try {
std::size_t dotPos = LogFilePath.find_last_of('.');

// create log file name/path and set default (*) logger
if (Tasks.size() > 1 && dotPos != std::string::npos) {
// create log file name/path and set default (*) logger. When more than
// one rank logs, append the rank id to keep per-rank files distinct.
if (NumLogging > 1 && dotPos != std::string::npos) {

NewLogFilePath = LogFilePath.substr(0, dotPos) + "_" +
std::to_string(TaskId) +
LogFilePath.substr(dotPos);
Expand Down
33 changes: 25 additions & 8 deletions components/omega/src/infra/Logging.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,19 @@
##__VA_ARGS__); \
_LOG_FLUSH

/// \def OMEGA_LOG_TASKS
/// A preprocessor variable that defines which tasks will be writing messages.
/// The default is to write a single log from task 0. If other tasks (or ALL)
/// are provided, log messages will be written to a corresponding log file with
/// the task number appended to the log file name.
#ifndef OMEGA_LOG_TASKS
#define OMEGA_LOG_TASKS 0
#endif
/// Logging task selection
/// Which MPI ranks write log files is controlled at runtime by the
/// OMEGA_LOG_TASKS environment variable, resolved against the Omega MPI
/// sub-communicator. When the variable is unset, logging defaults to the
/// master rank only. The selector accepts:
/// "*" - all ranks in the sub-communicator
/// "m" / "master" - the sub-communicator master rank only
/// "<n>" - a single rank
/// "0,2,4" - a comma-separated list of ranks
/// "0-3" - an inclusive range of ranks
/// "0,2-3" - any combination of lists and ranges
/// An invalid selector logs a warning on the master rank and falls back to
/// master-rank-only logging.

namespace OMEGA {

Expand Down Expand Up @@ -181,6 +186,18 @@ _PackLogMsg(const char *file, ///< [in] file where log called (cpp __FILE__)
const std::string &msg ///< [in] message text
);

/// Resolve a logging task selector string into the sorted, deduplicated list
/// of MPI ranks (relative to the Omega sub-communicator) that should log.
/// Accepts "*", "m"/"master", a single rank, comma lists, dash ranges, or any
/// combination of lists and ranges. On a malformed selector, sets Valid=false
/// and returns {MasterTask}.
std::vector<int>
_selectLogTasks(const std::string &Selector, ///< [in] raw selector string
I4 NumTasks, ///< [in] number of tasks in the sub-communicator
I4 MasterTask, ///< [in] master rank of the sub-communicator
bool &Valid ///< [out] false if the selector was malformed
);

} // namespace OMEGA

//===----------------------------------------------------------------------===//
Expand Down
Loading