diff --git a/pom.xml b/pom.xml index 619a2028..92cc8305 100644 --- a/pom.xml +++ b/pom.xml @@ -14,6 +14,8 @@ UTF-8 21 21 + + 6.2.10 @@ -50,6 +52,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + -parameters + -Xlint:deprecation + -Werror + + + + org.apache.maven.plugins maven-dependency-plugin @@ -102,66 +117,67 @@ - commons-configuration - commons-configuration - 1.9 - - - commons-logging - commons-logging - - + org.hibernate.orm + hibernate-core + 6.6.0.Final - org.slf4j - jcl-over-slf4j - 2.0.17 + org.mockito + mockito-junit-jupiter + 5.13.0 + test - commons-io - commons-io - 2.17.0 + org.junit.jupiter + junit-jupiter + 5.11.0 + test - code.google.com - jspf.core - 1.0.2 + com.h2database + h2 + 2.3.232 + test - org.hibernate - hibernate-core - 6.6.0.Final + org.springframework + spring-test + ${spring-framework.version} + test - org.apache.velocity - velocity - 1.7 + org.springframework + spring-context + ${spring-framework.version} - org.junit.jupiter - junit-jupiter-api - 5.11.3 - test + org.springframework + spring-jdbc + ${spring-framework.version} - org.mockito - mockito-junit-jupiter - 5.13.0 - test + org.springframework + spring-orm + ${spring-framework.version} - com.h2database - h2 - 2.3.232 - test + jakarta.annotation + jakarta.annotation-api + 3.0.0 + + + + com.zaxxer + HikariCP + 7.0.2 diff --git a/src/main/java/fr/insalyon/creatis/gasw/DatabaseConfiguration.java b/src/main/java/fr/insalyon/creatis/gasw/DatabaseConfiguration.java new file mode 100644 index 00000000..5d65a12f --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/DatabaseConfiguration.java @@ -0,0 +1,69 @@ +package fr.insalyon.creatis.gasw; + +import com.zaxxer.hikari.HikariDataSource; +import fr.insalyon.creatis.gasw.plugin.DatabasePlugin; +import jakarta.persistence.EntityManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; +import java.util.Properties; + +@Configuration +public class DatabaseConfiguration { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final DatabasePlugin dbPlugin; + + public DatabaseConfiguration(DatabasePlugin dbPlugin) { + this.dbPlugin = dbPlugin; + } + + @Bean + public DataSource dataSource() { + logger.info("Loading database plugin '{}' version '{}'", + dbPlugin.getName(), dbPlugin.getClass().getPackage().getImplementationVersion()); + + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dbPlugin.getDriverClass()); + dataSource.setUsername(dbPlugin.getUserName()); + dataSource.setPassword(dbPlugin.getPassword()); + dataSource.setJdbcUrl(dbPlugin.getConnectionUrl()); + return dataSource; + } + + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory(DataSource dataSource) { + LocalContainerEntityManagerFactoryBean emf = new LocalContainerEntityManagerFactoryBean(); + emf.setDataSource(dataSource); + emf.setPackagesToScan( + "fr.insalyon.creatis.gasw.bean", + "fr.insalyon.creatis.gasw.plugin" + ); + + emf.setJpaVendorAdapter(new HibernateJpaVendorAdapter()); + emf.setJpaProperties(hibernateProperties()); + return emf; + } + + private Properties hibernateProperties() { + Properties properties = new Properties(); + properties.setProperty("hibernate.default_schema", dbPlugin.getSchema()); + properties.setProperty("hibernate.hbm2ddl.auto", "update"); + properties.setProperty("hibernate.show_sql", "false"); + properties.setProperty("hibernate.format_sql", "false"); + return properties; + } + + @Bean + public PlatformTransactionManager transactionManager(EntityManagerFactory emf) { + return new JpaTransactionManager(emf); + } +} diff --git a/src/main/java/fr/insalyon/creatis/gasw/Gasw.java b/src/main/java/fr/insalyon/creatis/gasw/Gasw.java index cf0fa7c8..b8c69fb2 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/Gasw.java +++ b/src/main/java/fr/insalyon/creatis/gasw/Gasw.java @@ -32,70 +32,143 @@ */ package fr.insalyon.creatis.gasw; +import fr.insalyon.creatis.gasw.bean.SEEntryPoint; +import fr.insalyon.creatis.gasw.bean.SEEntryPointID; +import fr.insalyon.creatis.gasw.dao.DAOException; +import fr.insalyon.creatis.gasw.dao.SEEntryPointsDAO; import fr.insalyon.creatis.gasw.execution.ExecutorFactory; -import fr.insalyon.creatis.gasw.execution.FailOver; import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; + +import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +@Service public class Gasw { - private static final Logger logger = LoggerFactory.getLogger(Gasw.class); - private static Gasw instance; - private GaswNotification notification; - - /** - * Gets a default instance of GASW. - * - * @return Instance of GASW - */ - public synchronized static Gasw getInstance() throws GaswException { - - if (instance == null) { - instance = new Gasw(); + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final GaswConfiguration config; + private final GaswNotification gaswNotification; + private final ExecutorFactory executorFactory; + private final SEEntryPointsDAO seEntryPointsDAO; + private final List executorPlugins; + private final List listenerPlugins; + + public Gasw(GaswConfiguration config, GaswNotification gaswNotification, ExecutorFactory executorFactory, + SEEntryPointsDAO seEntryPointsDAO, List executorPlugins, List listenerPlugins) { + this.config = config; + this.gaswNotification = gaswNotification; + this.executorFactory = executorFactory; + this.seEntryPointsDAO = seEntryPointsDAO; + this.executorPlugins = executorPlugins; + this.listenerPlugins = listenerPlugins; + } + + @PostConstruct + public void init() throws GaswException { + if (config.isFailOverEnabled()) { + loadSEEntryPoints(); } - return instance; } - private Gasw() throws GaswException { - try { - logger.info("Initializing GASW."); - GaswConfiguration.getInstance().loadHibernate(); + // Log loaded plugins after all beans creation + @EventListener(ContextRefreshedEvent.class) + public void logPlugins() { + executorPlugins.forEach(p -> + logger.info("Loaded executor plugin '{}' version '{}'", + p.getName(), p.getClass().getPackage().getImplementationVersion())); + listenerPlugins.forEach(p -> + logger.info("Loaded listener plugin '{}' version '{}'", + p.getName(), p.getClass().getPackage().getImplementationVersion())); + } - notification = GaswNotification.getInstance(); + @PreDestroy + public void terminate() throws GaswException { + terminate(false); + } - } catch (IllegalArgumentException ex) { - throw new GaswException(ex); + public void terminate(boolean force) throws GaswException { + gaswNotification.terminate(); + for (ExecutorPlugin executorPlugin : executorPlugins) { + executorPlugin.terminate(force); } - } - public synchronized void setNotificationClient(Object client) { - notification.setClient(client); + for (ListenerPlugin listenerPlugin : listenerPlugins) { + listenerPlugin.terminate(); + } } - public synchronized String submit(GaswInput gaswInput) throws GaswException { - - ExecutorPlugin executor = ExecutorFactory.getExecutor(gaswInput); - executor.load(gaswInput); - return executor.submit(); + public void setNotificationClient(Object client) { + gaswNotification.setNotificationClient(client); } - public synchronized List getFinishedJobs() { - return notification.getFinishedJobs(); + public String submit(GaswInput gaswInput) throws GaswException { + return executorFactory.getExecutor().submit(gaswInput); } - public synchronized void waitForNotification() { - notification.waitForNotification(); + public List getFinishedJobs() { + return gaswNotification.getFinishedJobs(); } - public synchronized void terminate(boolean force) throws GaswException { - notification.terminate(); + private void loadSEEntryPoints() throws GaswException { + try { + logger.info("Loading SEs entry points."); + ProcessBuilder builder = new ProcessBuilder("lcg-info", "--list-service", + "--vo", config.getVoName(), "--attrs", "ServiceEndpoint"); + + builder.redirectErrorStream(true); + Process process = builder.start(); + + BufferedReader r = new BufferedReader(new InputStreamReader(process.getInputStream())); + String s = null; + StringBuilder cout = new StringBuilder(); + + while ((s = r.readLine()) != null) { + cout.append(s); + if (s.startsWith("- Service: httpg://")) { + try { + URI service = new URI(s.split(" ")[2]); + seEntryPointsDAO.add( + new SEEntryPoint(new SEEntryPointID( + service.getHost(), service.getPort()), + service.getPath())); + + } catch (URISyntaxException ex) { + logger.warn("Unable to read end point from: {}", s); + } catch (DAOException ex) { + if (!ex.getMessage().contains("duplicate key value")) { + logger.warn("Unable to save end point: {}", ex.getMessage()); + } + } + } + } + r.close(); + process.waitFor(); + + if (process.exitValue() != 0) { + logger.error(cout.toString()); + throw new GaswException("Unable to load SEs entry points."); + } + } catch (InterruptedException ex) { + logger.error("Error:", ex); + throw new GaswException(ex); - if (GaswConfiguration.getInstance().isFailOverEnabled()) { - FailOver.getInstance().terminate(); + } catch (IOException ex) { + logger.error("Error:", ex); + throw new GaswException(ex); } - - GaswConfiguration.getInstance().terminate(force); } } diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java b/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java index 68f7442d..b1822823 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswConfiguration.java @@ -32,362 +32,83 @@ */ package fr.insalyon.creatis.gasw; -import java.io.BufferedReader; import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.hibernate.SessionFactory; -import org.hibernate.boot.registry.StandardServiceRegistryBuilder; -import org.hibernate.cfg.Configuration; -import org.hibernate.service.ServiceRegistry; - -import fr.insalyon.creatis.gasw.bean.Data; -import fr.insalyon.creatis.gasw.bean.DataToReplicate; -import fr.insalyon.creatis.gasw.bean.Job; -import fr.insalyon.creatis.gasw.bean.JobMinorStatus; -import fr.insalyon.creatis.gasw.bean.Node; -import fr.insalyon.creatis.gasw.bean.NodeID; -import fr.insalyon.creatis.gasw.bean.SEEntryPoint; -import fr.insalyon.creatis.gasw.bean.SEEntryPointID; -import fr.insalyon.creatis.gasw.dao.DAOException; -import fr.insalyon.creatis.gasw.dao.DAOFactory; -import fr.insalyon.creatis.gasw.plugin.DatabasePlugin; -import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin; -import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; -import net.xeoh.plugins.base.PluginManager; -import net.xeoh.plugins.base.impl.PluginManagerFactory; -import net.xeoh.plugins.base.util.JSPFProperties; -import net.xeoh.plugins.base.util.PluginManagerUtil; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.beans.factory.annotation.Value; +@Configuration +@PropertySource("classpath:gasw.properties") public class GaswConfiguration { - - private static final Logger logger = LoggerFactory.getLogger(GaswConfiguration.class); - private static final String configDir = "./conf"; - private static final String configFile = "settings.conf"; - private static GaswConfiguration instance; - private static boolean strict = true; - private PropertiesConfiguration config; - private PluginManager pm; - // Properties - private String executionPath; - private String simulationID; + // Final properties + private final String executionPath = new File("").getAbsolutePath(); + private final String simulationID = executionPath.substring(executionPath.lastIndexOf("/") + 1); // Default Properties + @Value("${gasw.default.executor}") private String defaultExecutor; + @Value("${gasw.default.environment}") private String defaultEnvironment; + @Value("${gasw.default.background-script}") private String defaultBackgroundScript; + @Value("${gasw.default.requirements}") private String defaultRequirements; + @Value("${gasw.default.retry-count}") private int defaultRetryCount; + @Value("${gasw.default.timeout}") private int defaultTimeout; - private int defaultSleeptime; + @Value("${gasw.default.sleep-time}") + private int defaultSleeptimeSeconds; + @Value("${gasw.default.cpu-time}") private int defaultCPUTime; // Virtual Organization + @Value("${gasw.vo.name}") private String voName; + @Value("${gasw.vo.default-SE}") private String voDefaultSE; + @Value("${gasw.vo.use-close-SE}") private String voUseCloseSE; // Boutiques installation + @Value("${gasw.boutiques.bosh-CVMFS-path}") private String boshCVMFSPath; - private String singularityPath; - private String containersCVMFSPath; - private String udockerTag; - private String boutiquesProvenanceDir; + @Value("${gasw.boutiques.file-name}") private String boutiquesFileName; + @Value("${gasw.boutiques.provenance-dir}") + private String boutiquesProvenanceDir; // Containers stuff + @Value("${gasw.containers.runtime}") private String containersRuntime; + @Value("${gasw.containers.images-base-path}") private String containersImagesBasePath; + @Value("${gasw.containers.singularity-path}") + private String singularityPath; + @Value("${gasw.containers.CVMFS-path}") + private String containersCVMFSPath; + @Value("${gasw.containers.udocker-tag}") + private String udockerTag; // Failover Server + @Value("${gasw.failover.enabled}") private boolean failOverEnabled; + @Value("${gasw.failover.host}") private String failOverHost; + @Value("${gasw.failover.port}") private int failOverPort; + @Value("${gasw.failover.home}") private String failOverHome; + @Value("${gasw.failover.max-retry}") private int failOverMaxRetry; //MIN_AVG_DOWNLOAD_THROUGHPUT for the lcg-c* SEND_RECEIVE_TIMEOUT + @Value("${gasw.download.min-avg-throughput}") private int minAvgDownloadThroughput; // Minor Status Service + @Value("${gasw.minor-status.enabled}") private boolean minorStatusEnabled; // Others + @Value("${gasw.source.script}") private String sourceScript; - // Plugins - private List executorPluginsURI; - private List executorPlugins; - private String dbPluginURI; - private DatabasePlugin dbPlugin; - private List listenerPluginsURI; - private List listenerPlugins; - private SessionFactory sessionFactory; - - public static GaswConfiguration getInstance() throws GaswException { - if (instance == null) { - instance = new GaswConfiguration(); - } - return instance; - } - - public static void setStrict(boolean strict) { - GaswConfiguration.strict = strict; - } - - private GaswConfiguration() throws GaswException { - loadConfigurationFile(); - loadPlugins(); - - if (failOverEnabled) { - loadSEEntryPoints(); - } - } - - private void loadConfigurationFile() throws GaswException { - try { - executionPath = new File("").getAbsolutePath(); - simulationID = executionPath.substring(executionPath.lastIndexOf("/") + 1); - - config = new PropertiesConfiguration(new File(configDir + "/" + configFile)); - - defaultExecutor = config.getString(GaswConstants.LAB_DEFAULT_EXECUTOR, "Local"); - defaultEnvironment = config.getString(GaswConstants.LAB_DEFAULT_ENVIRONMENT, "\"\""); - defaultBackgroundScript = config.getString(GaswConstants.LAB_DEFAULT_BACKGROUD_SCRIPT, ""); - defaultRequirements = config.getString(GaswConstants.LAB_DEFAULT_REQUIREMENTS, ""); - defaultRetryCount = config.getInt(GaswConstants.LAB_DEFAULT_RETRY_COUNT, 5); - defaultTimeout = config.getInt(GaswConstants.LAB_DEFAULT_TIMEOUT, 100000); - defaultSleeptime = config.getInt(GaswConstants.LAB_DEFAULT_SLEEPTIME, 20) * 1000; - defaultCPUTime = config.getInt(GaswConstants.LAB_DEFAULT_CPUTIME, 1800); - - voName = config.getString(GaswConstants.LAB_VO_NAME, "biomed"); - voDefaultSE = config.getString(GaswConstants.LAB_VO_DEFAULT_SE, "SBG-disk"); - voUseCloseSE = config.getString(GaswConstants.LAB_VO_USE_CLOSE_SE, "\"true\""); - - boshCVMFSPath = config.getString(GaswConstants.LAB_BOSH_CVMFS_PATH, "\"/cvmfs/biomed.egi.eu/vip/virtualenv/bin\""); - singularityPath = config.getString(GaswConstants.LAB_SINGULARITY_PATH, "\"/cvmfs/dirac.egi.eu/dirac/v8.0.39/Linux-x86_64/bin\""); - containersCVMFSPath = config.getString(GaswConstants.LAB_CONTAINERS_CVMFS_PATH, "\"/cvmfs/biomed.egi.eu/vip/udocker/containers\""); - udockerTag = config.getString(GaswConstants.LAB_UDOCKER_TAG, "\"1.3.1\""); - boutiquesProvenanceDir = config.getString(GaswConstants.LAB_BOUTIQUES_PROV_DIR, "\"$HOME/.cache/boutiques/data\""); - boutiquesFileName = config.getString(GaswConstants.LAB_BOUTIQUES_FILE_NAME, "workflow.json"); - - containersRuntime = getRequiredString(config, GaswConstants.LAB_CONTAINERS_RUNTIME); - containersImagesBasePath = config.getString(GaswConstants.LAB_CONTAINERS_IMAGES_BASEPATH,"\"/cvmfs/biomed.egi.eu/vip/singularity\""); // path on singularity images. Should be provided by config - - failOverEnabled = config.getBoolean(GaswConstants.LAB_FAILOVER_ENABLED, false); - failOverHost = config.getString(GaswConstants.LAB_FAILOVER_HOST, "localhost"); - failOverPort = config.getInt(GaswConstants.LAB_FAILOVER_PORT, 8446); - failOverHome = config.getString(GaswConstants.LAB_FAILOVER_HOME, "/dpm/localhost/generated"); - failOverMaxRetry = config.getInt(GaswConstants.LAB_FAILOVER_RETRY, 3); - - minAvgDownloadThroughput = config.getInt(GaswConstants.LAB_MIN_AVG_DOWNLOAD_THROUGHPUT, 150); - - minorStatusEnabled = config.getBoolean(GaswConstants.LAB_MINORSTATUS_ENABLED, false); - - sourceScript = config.getString(GaswConstants.LAB_SOURCE_SCRIPT, ""); - - dbPluginURI = config.getString(GaswConstants.LAB_PLUGIN_DB, ""); - executorPluginsURI = config.getList(GaswConstants.LAB_PLUGIN_EXECUTOR); - listenerPluginsURI = config.getList(GaswConstants.LAB_PLUGIN_LISTENER); - - // Save - config.setProperty(GaswConstants.LAB_DEFAULT_EXECUTOR, defaultExecutor); - config.setProperty(GaswConstants.LAB_DEFAULT_ENVIRONMENT, defaultEnvironment); - config.setProperty(GaswConstants.LAB_DEFAULT_BACKGROUD_SCRIPT, defaultBackgroundScript); - config.setProperty(GaswConstants.LAB_DEFAULT_REQUIREMENTS, defaultRequirements); - config.setProperty(GaswConstants.LAB_DEFAULT_RETRY_COUNT, defaultRetryCount); - config.setProperty(GaswConstants.LAB_DEFAULT_TIMEOUT, defaultTimeout); - config.setProperty(GaswConstants.LAB_DEFAULT_SLEEPTIME, defaultSleeptime / 1000); - config.setProperty(GaswConstants.LAB_DEFAULT_CPUTIME, defaultCPUTime); - - config.setProperty(GaswConstants.LAB_VO_NAME, voName); - config.setProperty(GaswConstants.LAB_VO_DEFAULT_SE, voDefaultSE); - config.setProperty(GaswConstants.LAB_VO_USE_CLOSE_SE, voUseCloseSE); - - config.setProperty(GaswConstants.LAB_BOSH_CVMFS_PATH, boshCVMFSPath); - config.setProperty(GaswConstants.LAB_SINGULARITY_PATH, singularityPath); - config.setProperty(GaswConstants.LAB_CONTAINERS_CVMFS_PATH, containersCVMFSPath); - config.setProperty(GaswConstants.LAB_UDOCKER_TAG, udockerTag); - config.setProperty(GaswConstants.LAB_BOUTIQUES_PROV_DIR, boutiquesProvenanceDir); - config.setProperty(GaswConstants.LAB_BOUTIQUES_FILE_NAME, boutiquesFileName); - - config.setProperty(GaswConstants.LAB_FAILOVER_ENABLED, failOverEnabled); - config.setProperty(GaswConstants.LAB_FAILOVER_HOST, failOverHost); - config.setProperty(GaswConstants.LAB_FAILOVER_PORT, failOverPort); - config.setProperty(GaswConstants.LAB_FAILOVER_HOME, failOverHome); - - config.setProperty(GaswConstants.LAB_MIN_AVG_DOWNLOAD_THROUGHPUT, minAvgDownloadThroughput); - - config.setProperty(GaswConstants.LAB_MINORSTATUS_ENABLED, minorStatusEnabled); - - config.setProperty(GaswConstants.LAB_PLUGIN_DB, dbPluginURI); - config.setProperty(GaswConstants.LAB_PLUGIN_EXECUTOR, executorPluginsURI); - config.setProperty(GaswConstants.LAB_PLUGIN_LISTENER, listenerPluginsURI); - - new File(configDir).mkdirs(); - config.save(); - - } catch (ConfigurationException ex) { - logger.error("Error:", ex); - } - } - - private String getRequiredString(PropertiesConfiguration config, String key) throws GaswException { - if (config.getString(key) == null && strict) { - throw new GaswException("The property " + key + " should be present in configuration file!"); - } else { - return config.getString(key); - } - } - - private void loadPlugins() throws GaswException { - - final JSPFProperties props = new JSPFProperties(); - props.setProperty(PluginManager.class, "classpath.filter.default.pattern", "jre;com;javax;jena"); - - pm = PluginManagerFactory.createPluginManager(props); - - pm.addPluginsFrom(getAndLogPluginUri(dbPluginURI, "db")); - - for (Object o : executorPluginsURI) { - pm.addPluginsFrom(getAndLogPluginUri((String) o, "executor")); - } - - for (Object o : listenerPluginsURI) { - pm.addPluginsFrom(getAndLogPluginUri((String) o, "listener")); - } - - PluginManagerUtil pmu = new PluginManagerUtil(pm); - - dbPlugin = pmu.getPlugin(DatabasePlugin.class); - executorPlugins = (List) pmu.getPlugins(ExecutorPlugin.class); - listenerPlugins = (List) pmu.getPlugins(ListenerPlugin.class); - } - - private URI getAndLogPluginUri(String pluginPath, String pluginType) { - URI pluginUri = new File(pluginPath).toURI(); - logger.info("Loading {} plugin from {} (loaded URI : {})", pluginType, pluginPath, pluginUri); - return pluginUri; - } - - public void loadHibernate() throws GaswException { - logger.info("Loading database plugin '{}'.", dbPlugin.getName()); - dbPlugin.load(); - - Configuration cfg = new Configuration(); - - cfg.setProperty("hibernate.default_schema", dbPlugin.getSchema()); - cfg.setProperty("hibernate.connection.driver_class", dbPlugin.getDriverClass()); - cfg.setProperty("hibernate.connection.url", dbPlugin.getConnectionUrl()); - cfg.setProperty("hibernate.dialect", dbPlugin.getHibernateDialect()); - cfg.setProperty("hibernate.connection.username", dbPlugin.getUserName()); - cfg.setProperty("hibernate.connection.password", dbPlugin.getPassword()); - cfg.setProperty("hibernate.hbm2ddl.auto", "update"); - cfg.setProperty("hibernate.show_sql", false); - cfg.setProperty("hibernate.format_sql", false); - cfg.addAnnotatedClass(Data.class); - cfg.addAnnotatedClass(DataToReplicate.class); - cfg.addAnnotatedClass(Job.class); - cfg.addAnnotatedClass(JobMinorStatus.class); - cfg.addAnnotatedClass(Node.class); - cfg.addAnnotatedClass(NodeID.class); - cfg.addAnnotatedClass(SEEntryPoint.class); - cfg.addAnnotatedClass(SEEntryPointID.class); - - for (ExecutorPlugin executor : executorPlugins) { - for (Class c : executor.getPersistentClasses()) { - cfg.addAnnotatedClass(c); - } - } - for (ListenerPlugin listener : listenerPlugins) { - for (Class c : listener.getPersistentClasses()) { - cfg.addAnnotatedClass(c); - } - listener.load(); - } - - ServiceRegistry serviceRegistry = new StandardServiceRegistryBuilder().applySettings(cfg.getProperties()).build(); - sessionFactory = cfg.buildSessionFactory(serviceRegistry); - } - - private void loadSEEntryPoints() throws GaswException { - try { - logger.info("Loading SEs entry points."); - ProcessBuilder builder = new ProcessBuilder("lcg-info", "--list-service", - "--vo", voName, "--attrs", "ServiceEndpoint"); - - builder.redirectErrorStream(true); - Process process = builder.start(); - - BufferedReader r = new BufferedReader(new InputStreamReader(process.getInputStream())); - String s = null; - String cout = ""; - - while ((s = r.readLine()) != null) { - cout += s; - if (s.startsWith("- Service: httpg://")) { - try { - URI service = new URI(s.split(" ")[2]); - DAOFactory.getDAOFactory().getSEEntryPointDAO().add( - new SEEntryPoint(new SEEntryPointID( - service.getHost(), service.getPort()), - service.getPath())); - - } catch (URISyntaxException ex) { - logger.warn("Unable to read end point from: {}", s); - } catch (DAOException ex) { - if (!ex.getMessage().contains("duplicate key value")) { - logger.warn("Unable to save end point: {}", ex.getMessage()); - } - } - } - } - r.close(); - process.waitFor(); - - if (process.exitValue() != 0) { - logger.error(cout); - throw new GaswException("Unable to load SEs entry points."); - } - } catch (InterruptedException ex) { - logger.error("Error:", ex); - throw new GaswException(ex); - - } catch (IOException ex) { - logger.error("Error:", ex); - throw new GaswException(ex); - } - } - - public void terminate(boolean force) throws GaswException { - - for (ExecutorPlugin executorPlugin : executorPlugins) { - executorPlugin.terminate(force); - } - for (ListenerPlugin listenerPlugin : listenerPlugins) { - listenerPlugin.terminate(); - } - pm.shutdown(); - sessionFactory.close(); - } - - public PropertiesConfiguration getPropertiesConfiguration() { - return config; - } - - public List getExecutorPlugins() { - return executorPlugins; - } - - public List getListenerPlugins() { - return listenerPlugins; - } - - public SessionFactory getSessionFactory() { - return sessionFactory; - } public int getDefaultSleeptime() { - return defaultSleeptime; + return defaultSleeptimeSeconds; } public String getSimulationID() { @@ -486,11 +207,12 @@ public int getMinAvgDownloadThroughput() { return minAvgDownloadThroughput; } + public String getVoName() { + return voName; + } + public int getDefaultRetryCount() { return defaultRetryCount; } - public void setDbPlugin(DatabasePlugin databasePlugin) { - dbPlugin = databasePlugin; - } } \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java b/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java index 8d376ece..25d191b1 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswConstants.java @@ -38,39 +38,8 @@ */ public class GaswConstants { - // Configuration File Labels - public static final String LAB_DEFAULT_BACKGROUD_SCRIPT = "default.background.script"; - public static final String LAB_DEFAULT_CPUTIME = "default.cputime"; - public static final String LAB_DEFAULT_ENVIRONMENT = "default.environment"; - public static final String LAB_DEFAULT_EXECUTOR = "default.executor"; - public static final String LAB_DEFAULT_REQUIREMENTS = "default.requirements"; - public static final String LAB_DEFAULT_RETRY_COUNT = "default.retry.count"; - public static final String LAB_DEFAULT_SLEEPTIME = "default.sleeptime"; - public static final String LAB_DEFAULT_TIMEOUT = "default.timeout"; - public static final String LAB_FAILOVER_ENABLED = "failover.server.enabled"; - public static final String LAB_FAILOVER_HOME = "failover.server.home"; - public static final String LAB_FAILOVER_HOST = "failover.server.host"; - public static final String LAB_FAILOVER_PORT = "failover.server.port"; - public static final String LAB_FAILOVER_RETRY = "failover.max.retry"; - public static final String LAB_MINORSTATUS_ENABLED = "minorstatus.service.enabled"; - public static final String LAB_SOURCE_SCRIPT = "source.script"; - public static final String LAB_PLUGIN_DB = "plugin.db"; - public static final String LAB_PLUGIN_EXECUTOR = "plugin.executor"; - public static final String LAB_PLUGIN_LISTENER = "plugin.listener"; - public static final String LAB_VO_DEFAULT_SE = "vo.default.se"; - public static final String LAB_VO_NAME = "vo.name"; - public static final String LAB_VO_USE_CLOSE_SE = "vo.use.close.se"; - public static final String LAB_BOSH_CVMFS_PATH = "bosh.cvmfs.path"; - public static final String LAB_SINGULARITY_PATH = "singularity.path"; - public static final String LAB_CONTAINERS_CVMFS_PATH = "containers.cvmfs.path"; - public static final String LAB_UDOCKER_TAG = "udocker.tag"; - public static final String LAB_BOUTIQUES_PROV_DIR = "boutiques.provenance.dir"; - public static final String LAB_BOUTIQUES_FILE_NAME = "boutiques.filename"; - public static final String LAB_CONTAINERS_RUNTIME = "containers.runtime"; - public static final String LAB_CONTAINERS_IMAGES_BASEPATH = "containers.images.basepath"; // timeouts used in lcg-c* //public static final int SEND_RECEIVE_TIMEOUT = 900; - public static final String LAB_MIN_AVG_DOWNLOAD_THROUGHPUT = "min.avg.download.throughput"; public static final int CONNECT_TIMEOUT = 10; public static final int BDII_TIMEOUT = 10; public static final int SRM_TIMEOUT = 30; @@ -82,15 +51,12 @@ public class GaswConstants { public static final String PROVENANCE_ROOT = "./provenance"; public static final String CACHE_DIR = "${BASEDIR}/cache"; public static final String CACHE_FILE = "cache.txt"; - public static final String PROVENANCE_FILE = "provenance.json"; // Extensions public static final String OUT_EXT = ".out"; public static final String OUT_APP_EXT = ".app" + OUT_EXT; public static final String ERR_EXT = ".err"; public static final String ERR_APP_EXT = ".app" + ERR_EXT; public static final String PROVENANCE_EXT = ".provenance.json"; - // Environment Variables - public static final String ENV_EXECUTOR = "executor"; // moteur-lite constants public static final int numberOfReplicas = 1; public static final String INVOCATION_DIR = "./inv"; diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswLauncher.java b/src/main/java/fr/insalyon/creatis/gasw/GaswLauncher.java new file mode 100644 index 00000000..31cccdbe --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswLauncher.java @@ -0,0 +1,45 @@ +package fr.insalyon.creatis.gasw; + +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.support.ResourcePropertySource; + +import java.io.File; +import java.io.IOException; + +public class GaswLauncher { + + private static AnnotationConfigApplicationContext context; + + public static Gasw start() throws GaswException { + context = new AnnotationConfigApplicationContext(); + context.register(GaswSpringConfig.class); + loadExternalConfig(context.getEnvironment()); + context.refresh(); + return context.getBean(Gasw.class); + } + + public static void stop(boolean force) throws GaswException { + if (context != null) { + context.getBean(Gasw.class).terminate(force); + } + } + + private static void loadExternalConfig(ConfigurableEnvironment env) throws GaswException { + String home = System.getProperty("user.home"); + File dir = new File(home, ".gasw"); + if (!dir.exists()) return; + + File[] files = dir.listFiles((d, n) -> n.endsWith(".properties")); + if (files == null) return; + for (File f : files) { + try { + env.getPropertySources().addFirst( + new ResourcePropertySource(new FileSystemResource(f))); + } catch (IOException e) { + throw new GaswException("Failed to load configuration file " + f.getName(), e); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswNotification.java b/src/main/java/fr/insalyon/creatis/gasw/GaswNotification.java index 58de1ef0..e5510ba4 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/GaswNotification.java +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswNotification.java @@ -35,120 +35,73 @@ package fr.insalyon.creatis.gasw; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Queue; +import java.util.concurrent.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; -public class GaswNotification extends Thread { +@Service +public class GaswNotification { - private static final Logger logger = LoggerFactory.getLogger(GaswNotification.class); - private static GaswNotification instance; - private Notification notification; - private Object client; - private volatile List finishedJobs; - private volatile Map instanceErrorJobs; - private volatile boolean gettingOutputs; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public synchronized static GaswNotification getInstance() { + private final Queue finishedJobs; + private final Map instanceErrorJobs; - if (instance == null) { - instance = new GaswNotification(); - } - return instance; + private Object notificationClient; + private boolean stop = false; + + public GaswNotification() { + this.finishedJobs = new ConcurrentLinkedQueue<>(); + this.instanceErrorJobs = new ConcurrentHashMap<>(); } - private GaswNotification() { + @Scheduled(fixedDelayString = "#{@gaswConfiguration.defaultSleeptime / 2}", timeUnit = TimeUnit.SECONDS) + private void notifyIfReady() { + if (stop) return; + if (finishedJobs.isEmpty() || notificationClient == null) return; - this.finishedJobs = new ArrayList(); - this.gettingOutputs = false; - this.instanceErrorJobs = new HashMap<>(); + logger.debug("New tasks have finished execution. Notifying client..."); + Object client = notificationClient; + synchronized (client) { + client.notifyAll(); + } } - /** - * Sets the client to be notified when jobs are completed. - * - * @param client - */ - public void setClient(Object client) { - this.client = client; - notification = new Notification(); - notification.start(); + public void setNotificationClient(Object client) { + this.notificationClient = client; } - public synchronized void addFinishedJob(GaswOutput finishedJob) { - this.finishedJobs.add(finishedJob); + public void addFinishedJob(GaswOutput finishedJob) { + finishedJobs.add(finishedJob); } public List getFinishedJobs() { - gettingOutputs = true; - List outputsList = new ArrayList(); - - synchronized (finishedJobs) { - for (GaswOutput output : finishedJobs) { - outputsList.add(output); - } - finishedJobs = new ArrayList(); + List outputsList = new ArrayList<>(); + GaswOutput job; + while ((job = finishedJobs.poll()) != null) { + outputsList.add(job); } return outputsList; } - public synchronized void addErrorJob(GaswOutput errorJob) { - if (errorJob.getStdErr() != null) { - String instanceId = errorJob.getJobID(); - if (this.instanceErrorJobs.containsKey(instanceId)) { - this.instanceErrorJobs.replace(instanceId,errorJob); - } else { - this.instanceErrorJobs.put(instanceId,errorJob); - } - } - } + public void addErrorJob(GaswOutput errorJob) { + if (errorJob.getStdErr() == null) { return; } - public GaswOutput getGaswOutputFromLastFailedJob(String instanceId) { - if (this.instanceErrorJobs.containsKey(instanceId)) { - return this.instanceErrorJobs.get(instanceId); - } - return null; + instanceErrorJobs.merge(errorJob.getJobID(), errorJob, (oldValue, newValue) -> newValue); } - public void waitForNotification() { - gettingOutputs = false; + public GaswOutput getGaswOutputFromLastFailedJob(String instanceId) { + return instanceErrorJobs.get(instanceId); } - public void terminate() { - notification.terminate(); - } - - private class Notification extends Thread { - private boolean stop = false; - - @Override - public void run() { - - while (!stop) { - - if (!gettingOutputs && finishedJobs != null && !finishedJobs.isEmpty()) { - logger.debug("New tasks have finished execution. Notifying client..."); - synchronized (client) { - client.notify(); - } - } - try { - sleep(GaswConfiguration.getInstance().getDefaultSleeptime() / 2); - } catch (GaswException ex) { - logger.error("Error:", ex); - } catch (InterruptedException ex) { - logger.error("Error:", ex); - } - } - } - - public void terminate() { - stop = true; - } + stop = true; } } diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswOutput.java b/src/main/java/fr/insalyon/creatis/gasw/GaswOutput.java index 1b82278c..aa43f4ca 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/GaswOutput.java +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswOutput.java @@ -44,14 +44,14 @@ */ public class GaswOutput { - private String jobID; - private GaswExitCode exitCode; - private String exitMessage; - private Map uploadedResults; - private File appStdOut; - private File appStdErr; - private File stdOut; - private File stdErr; + private final String jobID; + private final GaswExitCode exitCode; + private final String exitMessage; + private final Map uploadedResults; + private final File appStdOut; + private final File appStdErr; + private final File stdOut; + private final File stdErr; /** * Creates an output object. diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswSpringConfig.java b/src/main/java/fr/insalyon/creatis/gasw/GaswSpringConfig.java new file mode 100644 index 00000000..b88419d0 --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswSpringConfig.java @@ -0,0 +1,45 @@ +package fr.insalyon.creatis.gasw; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +@Configuration +@ComponentScan({"fr.insalyon.creatis.gasw", "fr.insalyon.creatis.gasw.plugin"}) +@EnableTransactionManagement +@EnableScheduling +public class GaswSpringConfig { + + @Value("${gasw.scheduler.pool-size}") + private int poolSize; + + @Value("${gasw.scheduler.thread-name-prefix}") + private String threadNamePrefix; + + @Value("${gasw.scheduler.wait-for-tasks-on-shutdown}") + private boolean waitForTasksOnShutdown; + + @Value("${gasw.scheduler.await-termination-seconds}") + private int awaitTerminationSeconds; + + @Bean + public static PropertySourcesPlaceholderConfigurer propertySourcesPlaceholderConfigurer() { + return new PropertySourcesPlaceholderConfigurer(); + } + + @Bean + public ThreadPoolTaskScheduler taskScheduler() { + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setPoolSize(poolSize); + scheduler.setThreadNamePrefix(threadNamePrefix); + scheduler.setWaitForTasksToCompleteOnShutdown(waitForTasksOnShutdown); + scheduler.setAwaitTerminationSeconds(awaitTerminationSeconds); + scheduler.initialize(); + return scheduler; + } +} \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java b/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java index b822d6be..bfb06d73 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java +++ b/src/main/java/fr/insalyon/creatis/gasw/GaswUtil.java @@ -42,6 +42,7 @@ public class GaswUtil { private static final int[] times = {0, 10, 30, 45, 60, 90, 150, 300, 600, 900}; + private static final Pattern uriPattern = Pattern.compile("^\\w+:/{1,3}[^/]"); public static int sleep(Logger logger, String message, int index) throws InterruptedException { @@ -75,16 +76,38 @@ public static BufferedReader getBufferedReader(Process process) { return new BufferedReader(new InputStreamReader(process.getInputStream())); } - public static void closeProcess(Process process) throws IOException { - process.getOutputStream().close(); - process.getInputStream().close(); - process.getErrorStream().close(); - process = null; + public static void closeProcess(Logger logger, Process process) { + if (process == null) { + return; + } + + try { + process.getOutputStream().close(); + } catch (IOException ex) { + logger.warn("Failed to close process output stream", ex); + } + + try { + process.getInputStream().close(); + } catch (IOException ex) { + logger.warn("Failed to close process input stream", ex); + } + + try { + process.getErrorStream().close(); + } catch (IOException ex) { + logger.warn("Failed to close process error stream", ex); + } + + process.destroy(); } - private static final Pattern uriPattern = - Pattern.compile("^\\w+:/{1,3}[^/]"); public static boolean isUri(String s) { return uriPattern.matcher(s).find(); } + + public static String getBaseName(String name) { + int dot = name.lastIndexOf('.'); + return dot == -1 ? name : name.substring(0, dot); + } } \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/bean/Data.java b/src/main/java/fr/insalyon/creatis/gasw/bean/Data.java index 8e27ddba..47ba77ed 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/bean/Data.java +++ b/src/main/java/fr/insalyon/creatis/gasw/bean/Data.java @@ -39,6 +39,9 @@ * @author Rafael Ferreira da Silva */ @Entity +@NamedNativeQueries({ + @NamedNativeQuery(name = "Data.upsertData", query = "MERGE INTO Data (data_path, data_type) KEY(data_path) VALUES (?, ?)"), +}) @NamedQueries({ @NamedQuery(name = "Data.findByPath", query = "FROM Data d WHERE d.dataPath = :path") }) diff --git a/src/main/java/fr/insalyon/creatis/gasw/bean/Job.java b/src/main/java/fr/insalyon/creatis/gasw/bean/Job.java index 87c05de5..d9648987 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/bean/Job.java +++ b/src/main/java/fr/insalyon/creatis/gasw/bean/Job.java @@ -33,11 +33,11 @@ package fr.insalyon.creatis.gasw.bean; import fr.insalyon.creatis.gasw.execution.GaswStatus; +import jakarta.persistence.*; + import java.util.ArrayList; import java.util.Date; import java.util.List; -import jakarta.persistence.*; -import org.hibernate.annotations.Index; /** * @@ -45,21 +45,28 @@ */ @Entity @NamedQueries({ - @NamedQuery(name = "Job.findById", query = "FROM Job j WHERE j.id = :id"), - @NamedQuery(name = "Job.findByStatus", query = "FROM Job j WHERE j.status = :status"), - @NamedQuery(name = "Job.findByParameters", query = "FROM Job j WHERE j.parameters = :parameters"), - @NamedQuery(name = "Job.findActiveByInvocationID", query = "FROM Job j WHERE j.invocationID = :invocationID AND (status = :submitted OR status = :queued OR status = :running OR status = :kill OR status = :replicate OR status = :reschedule)"), - @NamedQuery(name = "Job.findFailedByInvocationID", query = "FROM Job j WHERE j.invocationID = :invocationID AND (status = :error OR status = :stalled OR status = :error_held OR status = :stalled_held)"), - @NamedQuery(name = "Job.getActive", query = "FROM Job j WHERE status = :submitted OR status = :queued OR status = :running OR status = :kill OR status = :replicate OR status = :reschedule"), - @NamedQuery(name = "Job.getCompletedJobsByInvocationID", query = "SELECT COUNT(j.id) FROM Job j WHERE j.invocationID = :invocationID AND status = :completed"), - @NamedQuery(name = "Job.getRunningByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :running OR status = :kill OR status = :replicate OR status = :reschedule)"), - @NamedQuery(name = "Job.getCompletedByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :completed)"), - @NamedQuery(name = "Job.getFailedByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :error OR status = :stalled OR status = :error_held OR status = :stalled_held)"), - @NamedQuery(name = "Job.getJobsByCommand", query = "FROM Job j WHERE j.command = :command"), - @NamedQuery(name = "Job.getJobsByFileName", query = "FROM Job j WHERE j.fileName = :fileName"), - @NamedQuery(name = "Job.getInvocationsByCommand", query = "SELECT DISTINCT j.invocationID FROM Job j WHERE j.command = :command") - }) -@Table(name = "Jobs") + @NamedQuery(name = "Job.findById", query = "FROM Job j WHERE j.id = :id"), + @NamedQuery(name = "Job.findByStatus", query = "FROM Job j WHERE j.status = :status"), + @NamedQuery(name = "Job.findByParameters", query = "FROM Job j WHERE j.parameters = :parameters"), + @NamedQuery(name = "Job.findActiveByInvocationID", query = "FROM Job j WHERE j.invocationID = :invocationID AND (status = :submitted OR status = :queued OR status = :running OR status = :replicate OR status = :reschedule)"), + @NamedQuery(name = "Job.findFailedByInvocationID", query = "FROM Job j WHERE j.invocationID = :invocationID AND (status = :error OR status = :stalled OR status = :error_held OR status = :stalled_held)"), + @NamedQuery(name = "Job.getActive", query = "FROM Job j WHERE status = :submitted OR status = :queued OR status = :running OR status = :replicate OR status = :reschedule"), + @NamedQuery(name = "Job.getCompletedJobsByInvocationID", query = "SELECT COUNT(j.id) FROM Job j WHERE j.invocationID = :invocationID AND status = :completed"), + @NamedQuery(name = "Job.getRunningByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :running OR status = :replicate OR status = :reschedule)"), + @NamedQuery(name = "Job.getCompletedByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :completed)"), + @NamedQuery(name = "Job.getFailedByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :error OR status = :stalled OR status = :error_held OR status = :stalled_held)"), + @NamedQuery(name = "Job.getJobsByCommand", query = "FROM Job j WHERE j.command = :command"), + @NamedQuery(name = "Job.getJobsByFileName", query = "FROM Job j WHERE j.fileName = :fileName"), + @NamedQuery(name = "Job.getInvocationsByCommand", query = "SELECT DISTINCT j.invocationID FROM Job j WHERE j.command = :command"), + @NamedQuery(name = "Job.getActiveJobsByCommand", query = "FROM Job j WHERE j.command = :command AND (status = :submitted OR status = :queued OR status = :running OR status = :replicate OR status = :reschedule)") + +}) +@Table( + name = "Jobs", + indexes = { + @Index(name = "paramIndex", columnList = "parameters"), + @Index(name = "invocationIndex", columnList = "invocation_id") +}) public class Job { private String id; @@ -100,11 +107,11 @@ public Job() { * @param executor */ public Job(String id, String simulationID, GaswStatus status, String command, - String fileName, String parameters, String executor) { + String fileName, String parameters, String executor) { this(id, simulationID, status, -1, "", null, null, null, null, null, - null, null, command, fileName, parameters, executor, - new ArrayList(), -1,null); + null, null, command, fileName, parameters, executor, + new ArrayList(), -1, null); } /** @@ -130,10 +137,10 @@ public Job(String id, String simulationID, GaswStatus status, String command, * @param diracSite */ public Job(String id, String simulationID, GaswStatus status, int exitCode, - String exitMessage, Date creation, Date queued, Date download, - Date running, Date upload, Date end, Node node, String command, - String fileName, String parameters, String executor, List data, - int invocationID, String diracSite) { + String exitMessage, Date creation, Date queued, Date download, + Date running, Date upload, Date end, Node node, String command, + String fileName, String parameters, String executor, List data, + int invocationID, String diracSite) { this.id = id; this.simulationID = simulationID; @@ -227,8 +234,8 @@ public void setExitMessage(String exitMessage) { @ManyToOne @JoinColumns({ - @JoinColumn(name = "node_site", referencedColumnName = "site"), - @JoinColumn(name = "node_name", referencedColumnName = "node_name") + @JoinColumn(name = "node_site", referencedColumnName = "site"), + @JoinColumn(name = "node_name", referencedColumnName = "node_name") }) public Node getNode() { return node; @@ -320,7 +327,6 @@ public void setFileName(String fileName) { } @Column(name = "parameters", length = 10000) - @Index(name = "paramIndex") public String getParameters() { return parameters; } @@ -363,9 +369,9 @@ public void setExecutor(String executor) { @ManyToMany(cascade = CascadeType.ALL) @JoinTable(name = "job_data", joinColumns = { - @JoinColumn(name = "id")}, + @JoinColumn(name = "id")}, inverseJoinColumns = { - @JoinColumn(name = "data_path")}) + @JoinColumn(name = "data_path")}) public List getData() { return data; } @@ -375,7 +381,6 @@ public void setData(List data) { } @Column(name = "invocation_id") - @Index(name = "invocationIndex") public int getInvocationID() { return invocationID; } diff --git a/src/main/java/fr/insalyon/creatis/gasw/bean/JobMinorStatus.java b/src/main/java/fr/insalyon/creatis/gasw/bean/JobMinorStatus.java index 554fcefd..4eb5fb6c 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/bean/JobMinorStatus.java +++ b/src/main/java/fr/insalyon/creatis/gasw/bean/JobMinorStatus.java @@ -37,7 +37,6 @@ import fr.insalyon.creatis.gasw.execution.GaswMinorStatus; import java.util.Date; import jakarta.persistence.*; -import org.hibernate.annotations.GenericGenerator; /** * @@ -74,8 +73,7 @@ public JobMinorStatus(Job job, GaswMinorStatus status, Date date) { } @Id - @GeneratedValue(generator = "increment") - @GenericGenerator(name = "increment", strategy = "increment") + @GeneratedValue(strategy = GenerationType.IDENTITY) public int getStatusId() { return statusId; } diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/DAOFactory.java b/src/main/java/fr/insalyon/creatis/gasw/dao/DAOFactory.java deleted file mode 100644 index cbb9ead6..00000000 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/DAOFactory.java +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright CNRS-CREATIS - * - * Rafael Silva - * rafael.silva@creatis.insa-lyon.fr - * http://www.rafaelsilva.com - * - * This software is a grid-enabled data-driven workflow manager and editor. - * - * This software is governed by the CeCILL license under French law and - * abiding by the rules of distribution of free software. You can use, - * modify and/ or redistribute the software under the terms of the CeCILL - * license as circulated by CEA, CNRS and INRIA at the following URL - * "http://www.cecill.info". - * - * As a counterpart to the access to the source code and rights to copy, - * modify and redistribute granted by the license, users are provided only - * with a limited warranty and the software's author, the holder of the - * economic rights, and the successive licensors have only limited - * liability. - * - * In this respect, the user's attention is drawn to the risks associated - * with loading, using, modifying and/or developing or reproducing the - * software by the user in light of its specific status of free software, - * that may mean that it is complicated to manipulate, and that also - * therefore means that it is reserved for developers and experienced - * professionals having in-depth computer knowledge. Users are therefore - * encouraged to load and test the software's suitability as regards their - * requirements in conditions enabling the security of their systems and/or - * data to be ensured and, more generally, to use and operate it in the - * same conditions as regards security. - * - * The fact that you are presently reading this means that you have had - * knowledge of the CeCILL license and that you accept its terms. - */ -package fr.insalyon.creatis.gasw.dao; - -/** - * - * @author Rafael Silva - */ -public abstract class DAOFactory { - - public synchronized static DAOFactory getDAOFactory() throws DAOException { - return HibernateDAOFactory.getInstance(); - } - - protected DAOFactory() {} - - public abstract void close(); - - public abstract JobDAO getJobDAO(); - - public abstract JobMinorStatusDAO getJobMinorStatusDAO(); - - public abstract NodeDAO getNodeDAO(); - - public abstract SEEntryPointsDAO getSEEntryPointDAO(); - - public abstract DataToReplicateDAO getDataToReplicateDAO(); -} diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/DataDAO.java b/src/main/java/fr/insalyon/creatis/gasw/dao/DataDAO.java new file mode 100644 index 00000000..1883957a --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/DataDAO.java @@ -0,0 +1,9 @@ +package fr.insalyon.creatis.gasw.dao; + +import fr.insalyon.creatis.gasw.bean.Data; + +public interface DataDAO { + + void upsertData(Data data) throws DAOException; + +} diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/HibernateDAOFactory.java b/src/main/java/fr/insalyon/creatis/gasw/dao/HibernateDAOFactory.java deleted file mode 100644 index 993cd8fb..00000000 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/HibernateDAOFactory.java +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright CNRS-CREATIS - * - * Rafael Silva - * rafael.silva@creatis.insa-lyon.fr - * http://www.rafaelsilva.com - * - * This software is a grid-enabled data-driven workflow manager and editor. - * - * This software is governed by the CeCILL license under French law and - * abiding by the rules of distribution of free software. You can use, - * modify and/ or redistribute the software under the terms of the CeCILL - * license as circulated by CEA, CNRS and INRIA at the following URL - * "http://www.cecill.info". - * - * As a counterpart to the access to the source code and rights to copy, - * modify and redistribute granted by the license, users are provided only - * with a limited warranty and the software's author, the holder of the - * economic rights, and the successive licensors have only limited - * liability. - * - * In this respect, the user's attention is drawn to the risks associated - * with loading, using, modifying and/or developing or reproducing the - * software by the user in light of its specific status of free software, - * that may mean that it is complicated to manipulate, and that also - * therefore means that it is reserved for developers and experienced - * professionals having in-depth computer knowledge. Users are therefore - * encouraged to load and test the software's suitability as regards their - * requirements in conditions enabling the security of their systems and/or - * data to be ensured and, more generally, to use and operate it in the - * same conditions as regards security. - * - * The fact that you are presently reading this means that you have had - * knowledge of the CeCILL license and that you accept its terms. - */ -package fr.insalyon.creatis.gasw.dao; - -import fr.insalyon.creatis.gasw.GaswConfiguration; -import fr.insalyon.creatis.gasw.GaswException; -import fr.insalyon.creatis.gasw.dao.hibernate.*; -import org.hibernate.SessionFactory; - -/** - * - * @author Rafael Silva - */ -public class HibernateDAOFactory extends DAOFactory { - - private static HibernateDAOFactory instance; - private SessionFactory sessionFactory; - public static HibernateDAOFactory getInstance() throws DAOException { - if (instance == null) { - instance = new HibernateDAOFactory(); - } - - return instance; - } - - private HibernateDAOFactory() throws DAOException { - - try { - this.sessionFactory = GaswConfiguration.getInstance().getSessionFactory(); - } catch (GaswException ex) { - throw new DAOException(ex); - } - } - - @Override - public void close() { - sessionFactory.close(); - } - - public SessionFactory getSessionFactory() { - return sessionFactory; - } - - @Override - public JobDAO getJobDAO() { - return new JobData(sessionFactory); - } - - @Override - public JobMinorStatusDAO getJobMinorStatusDAO() { - return new JobMinorStatusData(sessionFactory); - } - - @Override - public NodeDAO getNodeDAO() { - return new NodeData(sessionFactory); - } - - @Override - public SEEntryPointsDAO getSEEntryPointDAO() { - return new SEEntryPointData(sessionFactory); - } - - @Override - public DataToReplicateDAO getDataToReplicateDAO() { - return new DataToReplicateData(sessionFactory); - } -} diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/JobDAO.java b/src/main/java/fr/insalyon/creatis/gasw/dao/JobDAO.java index bf9dbc96..82f24381 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/JobDAO.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/JobDAO.java @@ -32,6 +32,7 @@ */ package fr.insalyon.creatis.gasw.dao; +import fr.insalyon.creatis.gasw.bean.Data; import fr.insalyon.creatis.gasw.bean.Job; import fr.insalyon.creatis.gasw.execution.GaswStatus; import java.util.List; @@ -73,4 +74,6 @@ public interface JobDAO { public List getByFileName(String filename) throws DAOException; public List getInvocationsByCommand(String command) throws DAOException; + + public List getActiveJobsByCommand(String command) throws DAOException; } diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataData.java new file mode 100644 index 00000000..4c6aa18e --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataData.java @@ -0,0 +1,41 @@ +package fr.insalyon.creatis.gasw.dao.hibernate; + +import fr.insalyon.creatis.gasw.bean.Data; +import fr.insalyon.creatis.gasw.dao.DAOException; +import fr.insalyon.creatis.gasw.dao.DataDAO; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +@Repository +public class DataData implements DataDAO { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @PersistenceContext + private EntityManager entityManager; + + /** + * Idempotent insert/update of a Data row keyed on data_path, to avoid + * unique constraint violations when multiple threads parse output for + * jobs referencing the same file path + */ + @Override + @Transactional + public void upsertData(Data data) throws DAOException { + try { + entityManager + .createNamedQuery("Data.upsertData") + .setParameter(1, data.getDataPath()) + .setParameter(2, data.getDataType().name()) + .executeUpdate(); + } catch (HibernateException ex) { + logger.error("Error while upserting data", ex); + throw new DAOException(ex); + } + } +} diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataToReplicateData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataToReplicateData.java index 2faab0ce..cc542e04 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataToReplicateData.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/DataToReplicateData.java @@ -37,32 +37,30 @@ import fr.insalyon.creatis.gasw.bean.DataToReplicate; import fr.insalyon.creatis.gasw.dao.DAOException; import fr.insalyon.creatis.gasw.dao.DataToReplicateDAO; - -import java.util.List; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hibernate.HibernateException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.util.List; + +@Repository public class DataToReplicateData implements DataToReplicateDAO { - private static final Logger logger = LoggerFactory.getLogger(DataToReplicate.class); - private SessionFactory sessionFactory; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public DataToReplicateData(SessionFactory sessionFactory) { - - this.sessionFactory = sessionFactory; - } - - @Override - public synchronized void add(DataToReplicate dataToReplicate) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(dataToReplicate); - session.getTransaction().commit(); + @PersistenceContext + private EntityManager entityManager; + @Override + @Transactional + public void add(DataToReplicate dataToReplicate) throws DAOException { + try { + entityManager + .merge(dataToReplicate); } catch (HibernateException ex) { logger.error("Error while adding", ex); throw new DAOException(ex); @@ -70,44 +68,36 @@ public synchronized void add(DataToReplicate dataToReplicate) throws DAOExceptio } @Override - public synchronized void update(DataToReplicate dataToReplicate) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(dataToReplicate); - session.getTransaction().commit(); - + @Transactional + public void update(DataToReplicate dataToReplicate) throws DAOException { + try { + entityManager + .merge(dataToReplicate); } catch (HibernateException ex) { logger.error("Error while updating", ex); throw new DAOException(ex); } } - - @Override - public synchronized void remove(DataToReplicate dataToReplicate) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.remove(dataToReplicate); - session.getTransaction().commit(); + @Override + @Transactional + public void remove(DataToReplicate dataToReplicate) throws DAOException { + try { + entityManager + .remove(dataToReplicate); } catch (HibernateException ex) { - logger.error("Error whice removing", ex); + logger.error("Error while removing", ex); throw new DAOException(ex); } } - - @Override - public synchronized List get() throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("DataToReplicate.list", DataToReplicate.class) - .list(); - session.getTransaction().commit(); - - return list; + @Override + @Transactional(readOnly = true) + public List get() throws DAOException { + try { + return entityManager + .createNamedQuery("DataToReplicate.list", DataToReplicate.class) + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving", ex); throw new DAOException(ex); diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobData.java index 06f66540..2757f405 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobData.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobData.java @@ -36,82 +36,68 @@ import fr.insalyon.creatis.gasw.dao.DAOException; import fr.insalyon.creatis.gasw.dao.JobDAO; import fr.insalyon.creatis.gasw.execution.GaswStatus; - -import java.util.List; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hibernate.HibernateException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +import java.util.List; + +@Repository public class JobData implements JobDAO { - private static final Logger logger = LoggerFactory.getLogger(JobData.class); - private SessionFactory sessionFactory; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public JobData(SessionFactory sessionFactory) { - this.sessionFactory = sessionFactory; - } + @PersistenceContext + private EntityManager entityManager; @Override + @Transactional public void add(Job job) throws DAOException { - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(job); - session.getTransaction().commit(); - + try { + entityManager + .merge(job); } catch (HibernateException ex) { logger.error("Error while adding", ex); throw new DAOException(ex); } } - /** - * Synchronized keyword for multi-threading context cause SQL Constraints violations issues - * due to unsynchronization of requests - */ @Override + @Transactional public void update(Job job) throws DAOException { - synchronized (sessionFactory) { - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(job); - session.getTransaction().commit(); - - } catch (HibernateException ex) { - logger.error("Error while updateing", ex); - throw new DAOException(ex); - } + try { + entityManager + .merge(job); + } catch (HibernateException ex) { + logger.error("Error while updating", ex); + throw new DAOException(ex); } } @Override + @Transactional public void remove(Job job) throws DAOException { - synchronized (sessionFactory) { - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.remove(job); - session.getTransaction().commit(); - - } catch (HibernateException ex) { - logger.error("Error while removing", ex); - throw new DAOException(ex); - } + try { + entityManager + .remove(job); + } catch (HibernateException ex) { + logger.error("Error while removing", ex); + throw new DAOException(ex); } } @Override + @Transactional(readOnly = true) public Job getJobByID(String id) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - Job job = session.createNamedQuery("Job.findById", Job.class) + try { + return entityManager + .createNamedQuery("Job.findById", Job.class) .setParameter("id", id) - .uniqueResult(); - session.getTransaction().commit(); - - return job; - + .getSingleResult(); } catch (HibernateException ex) { logger.error("Error while retrieving by ID", ex); throw new DAOException(ex); @@ -119,22 +105,17 @@ public Job getJobByID(String id) throws DAOException { } @Override + @Transactional(readOnly = true) public List getActiveJobs() throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getActive", Job.class) + try { + return entityManager + .createNamedQuery("Job.getActive", Job.class) .setParameter("submitted", GaswStatus.SUCCESSFULLY_SUBMITTED) .setParameter("queued", GaswStatus.QUEUED) .setParameter("running", GaswStatus.RUNNING) - .setParameter("kill", GaswStatus.KILL) .setParameter("replicate", GaswStatus.REPLICATE) .setParameter("reschedule", GaswStatus.RESCHEDULE) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving actives jobs", ex); throw new DAOException(ex); @@ -142,16 +123,13 @@ public List getActiveJobs() throws DAOException { } @Override + @Transactional(readOnly = true) public List getJobs(GaswStatus status) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.findByStatus", Job.class) - .setParameter("status", status).list(); - session.getTransaction().commit(); - - return list; - + try { + return entityManager + .createNamedQuery("Job.findByStatus", Job.class) + .setParameter("status", status) + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving jobs", ex); throw new DAOException(ex); @@ -159,42 +137,33 @@ public List getJobs(GaswStatus status) throws DAOException { } @Override + @Transactional(readOnly = true) public long getNumberOfCompletedJobsByInvocationID(int invocationID) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - long completedJobs = session.createNamedQuery("Job.getCompletedJobsByInvocationID", Long.class) + try { + return entityManager + .createNamedQuery("Job.getCompletedJobsByInvocationID", Long.class) .setParameter("invocationID", invocationID) .setParameter("completed", GaswStatus.COMPLETED) - .uniqueResult(); - session.getTransaction().commit(); - - return completedJobs; - + .getSingleResult(); } catch (HibernateException ex) { - logger.error("Error while retrieving completed jobs by invocation ID",ex); + logger.error("Error while retrieving completed jobs by invocation ID", ex); throw new DAOException(ex); } } @Override + @Transactional(readOnly = true) public List getActiveJobsByInvocationID(int invocationID) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.findActiveByInvocationID", Job.class) + try { + return entityManager + .createNamedQuery("Job.findActiveByInvocationID", Job.class) .setParameter("invocationID", invocationID) .setParameter("submitted", GaswStatus.SUCCESSFULLY_SUBMITTED) .setParameter("queued", GaswStatus.QUEUED) .setParameter("running", GaswStatus.RUNNING) - .setParameter("kill", GaswStatus.KILL) .setParameter("replicate", GaswStatus.REPLICATE) .setParameter("reschedule", GaswStatus.RESCHEDULE) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving actives jobs by invocation ID", ex); throw new DAOException(ex); @@ -202,21 +171,17 @@ public List getActiveJobsByInvocationID(int invocationID) throws DAOExcepti } @Override + @Transactional(readOnly = true) public List getFailedJobsByInvocationID(int invocationID) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.findFailedByInvocationID", Job.class) + try { + return entityManager + .createNamedQuery("Job.findFailedByInvocationID", Job.class) .setParameter("invocationID", invocationID) .setParameter("error", GaswStatus.ERROR) .setParameter("stalled", GaswStatus.STALLED) .setParameter("error_held", GaswStatus.ERROR_HELD) .setParameter("stalled_held", GaswStatus.STALLED_HELD) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving failed jobs by invocation ID", ex); throw new DAOException(ex); @@ -224,21 +189,16 @@ public List getFailedJobsByInvocationID(int invocationID) throws DAOExcepti } @Override + @Transactional(readOnly = true) public List getRunningByCommand(String command) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getRunningByCommand", Job.class) + try { + return entityManager + .createNamedQuery("Job.getRunningByCommand", Job.class) .setParameter("command", command) .setParameter("running", GaswStatus.RUNNING) - .setParameter("kill", GaswStatus.KILL) .setParameter("replicate", GaswStatus.REPLICATE) .setParameter("reschedule", GaswStatus.RESCHEDULE) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving running jobs by command", ex); throw new DAOException(ex); @@ -246,18 +206,14 @@ public List getRunningByCommand(String command) throws DAOException { } @Override + @Transactional(readOnly = true) public List getCompletedByCommand(String command) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getCompletedByCommand", Job.class) + try { + return entityManager + .createNamedQuery("Job.getCompletedByCommand", Job.class) .setParameter("command", command) .setParameter("completed", GaswStatus.COMPLETED) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving completed jobs by command", ex); throw new DAOException(ex); @@ -265,17 +221,13 @@ public List getCompletedByCommand(String command) throws DAOException { } @Override + @Transactional(readOnly = true) public List getByParameters(String parameters) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.findByParameters", Job.class) + try { + return entityManager + .createNamedQuery("Job.findByParameters", Job.class) .setParameter("parameters", parameters) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving jobs by parameters", ex); throw new DAOException(ex); @@ -283,21 +235,17 @@ public List getByParameters(String parameters) throws DAOException { } @Override + @Transactional(readOnly = true) public List getFailedByCommand(String command) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getFailedByCommand", Job.class) + try { + return entityManager + .createNamedQuery("Job.getFailedByCommand", Job.class) .setParameter("command", command) .setParameter("error", GaswStatus.ERROR) .setParameter("stalled", GaswStatus.STALLED) .setParameter("error_held", GaswStatus.ERROR_HELD) .setParameter("stalled_held", GaswStatus.STALLED_HELD) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving failed jobs by command", ex); throw new DAOException(ex); @@ -305,17 +253,13 @@ public List getFailedByCommand(String command) throws DAOException { } @Override + @Transactional(readOnly = true) public List getJobsByCommand(String command) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getJobsByCommand", Job.class) + try { + return entityManager + .createNamedQuery("Job.getJobsByCommand", Job.class) .setParameter("command", command) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving jobs by command", ex); throw new DAOException(ex); @@ -323,17 +267,13 @@ public List getJobsByCommand(String command) throws DAOException { } @Override + @Transactional(readOnly = true) public List getInvocationsByCommand(String command) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getInvocationsByCommand", Integer.class) + try { + return entityManager + .createNamedQuery("Job.getInvocationsByCommand", Integer.class) .setParameter("command", command) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving invocations by command", ex); throw new DAOException(ex); @@ -341,17 +281,32 @@ public List getInvocationsByCommand(String command) throws DAOException } @Override - public List getByFileName(String filename) throws DAOException { + @Transactional(readOnly = true) + public List getActiveJobsByCommand(String command) throws DAOException { + try { + return entityManager + .createNamedQuery("Job.getActiveJobsByCommand", Job.class) + .setParameter("command", command) + .setParameter("submitted", GaswStatus.SUCCESSFULLY_SUBMITTED) + .setParameter("queued", GaswStatus.QUEUED) + .setParameter("running", GaswStatus.RUNNING) + .setParameter("replicate", GaswStatus.REPLICATE) + .setParameter("reschedule", GaswStatus.RESCHEDULE) + .getResultList(); + } catch (HibernateException ex) { + logger.error("Error while retrieving active jobs by command", ex); + throw new DAOException(ex); + } + } - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("Job.getJobsByFileName", Job.class) + @Override + @Transactional(readOnly = true) + public List getByFileName(String filename) throws DAOException { + try { + return entityManager + .createNamedQuery("Job.getJobsByFileName", Job.class) .setParameter("fileName", filename) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving jobs by filename", ex); throw new DAOException(ex); diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobMinorStatusData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobMinorStatusData.java index b1b4ece1..3965ab38 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobMinorStatusData.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/JobMinorStatusData.java @@ -38,31 +38,30 @@ import fr.insalyon.creatis.gasw.dao.DAOException; import fr.insalyon.creatis.gasw.dao.JobMinorStatusDAO; import fr.insalyon.creatis.gasw.execution.GaswMinorStatus; -import java.util.List; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hibernate.HibernateException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; -public class JobMinorStatusData implements JobMinorStatusDAO { +import java.util.List; - private static final Logger logger = LoggerFactory.getLogger(JobMinorStatusData.class); - private SessionFactory sessionFactory; +@Repository +public class JobMinorStatusData implements JobMinorStatusDAO { - public JobMinorStatusData(SessionFactory sessionFactory) { + private final Logger logger = LoggerFactory.getLogger(getClass()); - this.sessionFactory = sessionFactory; - } + @PersistenceContext + private EntityManager entityManager; @Override + @Transactional public void add(JobMinorStatus jobMinorStatus) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(jobMinorStatus); - session.getTransaction().commit(); - + try { + entityManager + .merge(jobMinorStatus); } catch (HibernateException ex) { logger.error("Error while adding", ex); throw new DAOException(ex); @@ -70,20 +69,16 @@ public void add(JobMinorStatus jobMinorStatus) throws DAOException { } @Override + @Transactional(readOnly = true) public List getCheckpoints(String jobID) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("MinorStatus.findCheckpointById", JobMinorStatus.class) + try { + return entityManager + .createNamedQuery("MinorStatus.findCheckpointById", JobMinorStatus.class) .setParameter("jobId", jobID) .setParameter("checkpointInit", GaswMinorStatus.CheckPoint_Init) .setParameter("checkpointUpload", GaswMinorStatus.CheckPoint_Upload) .setParameter("checkpointEnd", GaswMinorStatus.CheckPoint_Upload) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving checkpoints", ex); throw new DAOException(ex); @@ -91,11 +86,11 @@ public List getCheckpoints(String jobID) throws DAOException { } @Override + @Transactional(readOnly = true) public List getExecutionMinorStatus(String jobID) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("MinorStatus.findExecutionById", JobMinorStatus.class) + try { + return entityManager + .createNamedQuery("MinorStatus.findExecutionById", JobMinorStatus.class) .setParameter("jobId", jobID) .setParameter("start", GaswMinorStatus.Started) .setParameter("background", GaswMinorStatus.Background) @@ -103,11 +98,7 @@ public List getExecutionMinorStatus(String jobID) throws DAOExce .setParameter("application", GaswMinorStatus.Application) .setParameter("output", GaswMinorStatus.Outputs) .setParameter("finished", GaswMinorStatus.Finished) - .list(); - session.getTransaction().commit(); - - return list; - + .getResultList(); } catch (HibernateException ex) { logger.error("Error while retrieving minorstatus", ex); throw new DAOException(ex); @@ -115,20 +106,18 @@ public List getExecutionMinorStatus(String jobID) throws DAOExce } @Override - public long getDateDiff(String jobID, GaswMinorStatus start, - GaswMinorStatus end) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - List list = session.createNamedQuery("MinorStatus.dateDiff", JobMinorStatus.class) + @Transactional(readOnly = true) + public long getDateDiff(String jobID, GaswMinorStatus start, + GaswMinorStatus end) throws DAOException { + try { + List list = entityManager + .createNamedQuery("MinorStatus.dateDiff", JobMinorStatus.class) .setParameter("jobId", jobID) .setParameter("start", start) .setParameter("end", end) - .list(); - session.close(); - - return Math.abs(list.get(1).getDate().getTime() - list.get(0).getDate().getTime()); + .getResultList(); + return Math.abs(list.get(1).getDate().getTime() - list.get(0).getDate().getTime()); } catch (HibernateException ex) { logger.error("Error while retrieving date diff", ex); throw new DAOException(ex); diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/NodeData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/NodeData.java index d1898c81..a6756f08 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/NodeData.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/NodeData.java @@ -37,30 +37,28 @@ import fr.insalyon.creatis.gasw.bean.Node; import fr.insalyon.creatis.gasw.dao.DAOException; import fr.insalyon.creatis.gasw.dao.NodeDAO; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hibernate.HibernateException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +@Repository public class NodeData implements NodeDAO { - private static final Logger logger = LoggerFactory.getLogger(NodeData.class); - private SessionFactory sessionFactory; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public NodeData(SessionFactory sessionFactory) { - - this.sessionFactory = sessionFactory; - } + @PersistenceContext + private EntityManager entityManager; @Override + @Transactional public void add(Node node) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(node); - session.getTransaction().commit(); - + try { + entityManager + .merge(node); } catch (HibernateException ex) { logger.error("Error while adding", ex); throw new DAOException(ex); @@ -68,18 +66,14 @@ public void add(Node node) throws DAOException { } @Override + @Transactional(readOnly = true) public Node getNodeBySiteAndNodeName(String site, String nodeName) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - Node node = session.createNamedQuery("Node.findBySiteAndNodeName", Node.class) + try { + return entityManager + .createNamedQuery("Node.findBySiteAndNodeName", Node.class) .setParameter("siteName", site) .setParameter("nodeName", nodeName) - .uniqueResult(); - session.getTransaction().commit(); - - return node; - + .getSingleResult(); } catch (HibernateException ex) { logger.error("Error while retrieving", ex); throw new DAOException(ex); diff --git a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/SEEntryPointData.java b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/SEEntryPointData.java index 32e17f35..5dbd7adf 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/SEEntryPointData.java +++ b/src/main/java/fr/insalyon/creatis/gasw/dao/hibernate/SEEntryPointData.java @@ -37,30 +37,28 @@ import fr.insalyon.creatis.gasw.bean.SEEntryPoint; import fr.insalyon.creatis.gasw.dao.DAOException; import fr.insalyon.creatis.gasw.dao.SEEntryPointsDAO; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import org.hibernate.HibernateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.hibernate.HibernateException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; +@Repository public class SEEntryPointData implements SEEntryPointsDAO { - private static final Logger logger = LoggerFactory.getLogger(SEEntryPointData.class); - private SessionFactory sessionFactory; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public SEEntryPointData(SessionFactory sessionFactory) { - - this.sessionFactory = sessionFactory; - } - - @Override - public synchronized void add(SEEntryPoint seEntryPoint) throws DAOException { - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - session.merge(seEntryPoint); - session.getTransaction().commit(); + @PersistenceContext + private EntityManager entityManager; + @Override + @Transactional + public void add(SEEntryPoint seEntryPoint) throws DAOException { + try { + entityManager + .merge(seEntryPoint); } catch (HibernateException ex) { logger.error("Error while adding", ex); throw new DAOException(ex); @@ -68,17 +66,13 @@ public synchronized void add(SEEntryPoint seEntryPoint) throws DAOException { } @Override - public synchronized SEEntryPoint getByHostName(String hostname) throws DAOException { - - - try (Session session = sessionFactory.openSession()) { - session.beginTransaction(); - SEEntryPoint entryPoint = session.createNamedQuery("EntryPoints.findByHostname", SEEntryPoint.class) - .setParameter("hostname", hostname).uniqueResult(); - session.getTransaction().commit(); - - return entryPoint; - + @Transactional(readOnly = true) + public SEEntryPoint getByHostName(String hostname) throws DAOException { + try { + return entityManager + .createNamedQuery("EntryPoints.findByHostname", SEEntryPoint.class) + .setParameter("hostname", hostname) + .getSingleResult(); } catch (HibernateException ex) { logger.error("Error while retrieving", ex); throw new DAOException(ex); diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/ExecutorFactory.java b/src/main/java/fr/insalyon/creatis/gasw/execution/ExecutorFactory.java index 0f44bee7..f97b97da 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/execution/ExecutorFactory.java +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/ExecutorFactory.java @@ -34,20 +34,36 @@ import fr.insalyon.creatis.gasw.GaswConfiguration; import fr.insalyon.creatis.gasw.GaswException; -import fr.insalyon.creatis.gasw.GaswInput; import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; -public class ExecutorFactory { +import java.util.List; - public static ExecutorPlugin getExecutor(GaswInput gaswInput) throws GaswException { +@Component +public class ExecutorFactory implements ApplicationContextAware { - String executorName = GaswConfiguration.getInstance().getDefaultExecutor(); + private final GaswConfiguration config; + private ApplicationContext applicationContext; - for (ExecutorPlugin executor : GaswConfiguration.getInstance().getExecutorPlugins()) { - if (executor.getName().equalsIgnoreCase(executorName)) { - return executor; - } - } - throw new GaswException("There is no executor available for '" + executorName + "'."); + public ExecutorFactory(GaswConfiguration config) { + this.config = config; + } + + public ExecutorPlugin getExecutor() throws GaswException { + String executorName = config.getDefaultExecutor(); + return applicationContext.getBeansOfType(ExecutorPlugin.class).values() + .stream() + .filter(plugin -> plugin.getName().equalsIgnoreCase(executorName)) + .findFirst() + .orElseThrow(() -> + new GaswException("There is no executor available for '" + executorName + "'.")); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; } } \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/FailOver.java b/src/main/java/fr/insalyon/creatis/gasw/execution/FailOver.java index a5d4508a..aee4d8d8 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/execution/FailOver.java +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/FailOver.java @@ -40,7 +40,6 @@ import fr.insalyon.creatis.gasw.bean.DataToReplicate; import fr.insalyon.creatis.gasw.bean.SEEntryPoint; import fr.insalyon.creatis.gasw.dao.DAOException; -import fr.insalyon.creatis.gasw.dao.DAOFactory; import fr.insalyon.creatis.gasw.dao.DataToReplicateDAO; import java.io.BufferedReader; import java.io.Closeable; @@ -52,66 +51,56 @@ import java.util.Date; import java.util.List; import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import fr.insalyon.creatis.gasw.dao.SEEntryPointsDAO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; -public class FailOver extends Thread { +@Service +public class FailOver { - private static final Logger logger = LoggerFactory.getLogger(FailOver.class); - private static FailOver instance; - private volatile boolean stop = false; - private DataToReplicateDAO dataToReplicateDAO; + private final Logger logger = LoggerFactory.getLogger(getClass()); - public synchronized static FailOver getInstance() { - if (instance == null) { - instance = new FailOver(); - instance.start(); - } - return instance; - } + private final GaswConfiguration config; + private final DataToReplicateDAO dataToReplicateDAO; + private final SEEntryPointsDAO seEntryPointDAO; - private FailOver() { - try { - dataToReplicateDAO = DAOFactory.getDAOFactory().getDataToReplicateDAO(); - } catch (DAOException ex) { - logger.error("Unable to start Fail Over thread."); - } + public FailOver(GaswConfiguration config, DataToReplicateDAO dataToReplicateDAO, SEEntryPointsDAO seEntryPointDAO) { + this.config = config; + this.dataToReplicateDAO = dataToReplicateDAO; + this.seEntryPointDAO = seEntryPointDAO; } - @Override - public void run() { + @Scheduled(fixedDelayString = "${gasw.default.sleep-time}", timeUnit = TimeUnit.SECONDS) + private void run() { + if (!config.isFailOverEnabled()) { return; } try { - while (!stop) { - - for (DataToReplicate data : dataToReplicateDAO.get()) { - try { - replicate(data.getUrl()); - dataToReplicateDAO.remove(data); + for (DataToReplicate data : dataToReplicateDAO.get()) { + try { + replicate(data.getUrl()); + dataToReplicateDAO.remove(data); - } catch (GaswException ex) { + } catch (GaswException ex) { - if (data.getRetries() + 1 < GaswConfiguration.getInstance().getFailOverMaxRetry()) { - data.setRetries(data.getRetries() + 1); - data.setEventDate(new Date()); - dataToReplicateDAO.update(data); - } else { - logger.warn("Achieved data max attempts to reply '{}'.", data.getUrl().getPath()); - dataToReplicateDAO.remove(data); - } + if (data.getRetries() + 1 < config.getFailOverMaxRetry()) { + data.setRetries(data.getRetries() + 1); + data.setEventDate(new Date()); + dataToReplicateDAO.update(data); + } else { + logger.warn("Achieved data max attempts to reply '{}'.", data.getUrl().getPath()); + dataToReplicateDAO.remove(data); } } - Thread.sleep(GaswConfiguration.getInstance().getDefaultSleeptime()); } } catch (DAOException ex) { - // do nothing - } catch (GaswException ex) { - // do nothing - } catch (InterruptedException ex) { - logger.error("InterruptedException: ",ex); + logger.error("DAOException: ",ex); } } - public synchronized void addData(URI uri) { + public void addData(URI uri) { try { String scheme = uri.getScheme(); if (scheme == null || (!scheme.equalsIgnoreCase("file") @@ -124,22 +113,18 @@ public synchronized void addData(URI uri) { } } - public synchronized void addData(List uris) { + public void addData(List uris) { for (URI uri : uris) { addData(uri); } } - public synchronized void terminate() { - this.stop = true; - } - private void replicate(URI uri) throws GaswException { List replicas = getReplicas(uri); for (URI replica : replicas) { - if (replica.getHost().equals(GaswConfiguration.getInstance().getFailOverHost())) { + if (replica.getHost().equals(config.getFailOverHost())) { return; } } @@ -176,9 +161,7 @@ private void replicate(URI uri) throws GaswException { } catch (IOException ex) { logger.warn("IOException:", ex); } finally { - if (process != null) { - close(process); - } + GaswUtil.closeProcess(logger, process); if (br != null) { try { br.close(); @@ -225,9 +208,7 @@ private List getReplicas(URI uri) throws GaswException { throw new GaswException(ex); } finally { - if (process != null) { - close(process); - } + GaswUtil.closeProcess(logger, process); if (br != null) { try { br.close(); @@ -239,19 +220,19 @@ private List getReplicas(URI uri) throws GaswException { return replicas; } - private String getDestinationSURL() throws GaswException { + private String getDestinationSURL() { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - return "srm://" + GaswConfiguration.getInstance().getFailOverHost() - + ":" + GaswConfiguration.getInstance().getFailOverPort() - + "/srm/managerv2?SFN=" + GaswConfiguration.getInstance().getFailOverHome() + return "srm://" + config.getFailOverHost() + + ":" + config.getFailOverPort() + + "/srm/managerv2?SFN=" + config.getFailOverHome() + "/" + sdf.format(new Date()) + "/file-" + UUID.randomUUID(); } private String[] getSourceTypeAndSURL(String host, String path) throws DAOException { - SEEntryPoint ep = DAOFactory.getDAOFactory().getSEEntryPointDAO().getByHostName(host); + SEEntryPoint ep = seEntryPointDAO.getByHostName(host); String[] source = new String[]{ ep.getHome().contains("managerv1") ? "srmv1" : "srmv2", "srm://" + ep.getId().getHostname() + ":" + ep.getId().getPort() @@ -260,23 +241,4 @@ private String[] getSourceTypeAndSURL(String host, String path) throws DAOExcept return source; } - - private void close(Process process) { - - close(process.getOutputStream()); - close(process.getInputStream()); - close(process.getErrorStream()); - process.destroy(); - } - - private void close(Closeable c) { - - if (c != null) { - try { - c.close(); - } catch (IOException ex) { - // ignored - } - } - } } diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswMonitor.java b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswMonitor.java index 53a680c1..58952ed3 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswMonitor.java +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswMonitor.java @@ -36,50 +36,48 @@ import fr.insalyon.creatis.gasw.GaswException; import fr.insalyon.creatis.gasw.bean.Job; import fr.insalyon.creatis.gasw.dao.DAOException; -import fr.insalyon.creatis.gasw.dao.DAOFactory; import fr.insalyon.creatis.gasw.dao.JobDAO; -import fr.insalyon.creatis.gasw.dao.NodeDAO; import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; import java.util.Date; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; -public abstract class GaswMonitor extends Thread { +public abstract class GaswMonitor { - private static final Logger logger = LoggerFactory.getLogger(GaswMonitor.class); + private final Logger logger = LoggerFactory.getLogger(getClass()); - private volatile static int INVOCATION_ID = 1; - protected JobDAO jobDAO; - protected NodeDAO nodeDAO; + private final GaswConfiguration config; + private final JobDAO jobDAO; + private final List listenerPlugins; - protected GaswMonitor() { - try { - jobDAO = DAOFactory.getDAOFactory().getJobDAO(); - nodeDAO = DAOFactory.getDAOFactory().getNodeDAO(); + private static final AtomicInteger INVOCATION_ID = new AtomicInteger(1); - } catch (DAOException ex) { - // do nothing - } + public GaswMonitor(GaswConfiguration config, JobDAO jobDAO, List listenerPlugins) { + this.config = config; + this.jobDAO = jobDAO; + this.listenerPlugins = listenerPlugins; } - protected synchronized void add(Job job) throws GaswException { + @Transactional + protected void add(Job job) throws GaswException { try { // Defining invocation ID - List list = jobDAO.getByFileName(job.getFileName()); if (!list.isEmpty()) { job.setInvocationID(list.get(0).getInvocationID()); } else { - job.setInvocationID(INVOCATION_ID++); + job.setInvocationID(INVOCATION_ID.getAndIncrement()); } job.setCreation(new Date()); jobDAO.add(job); // Listeners notification - for (ListenerPlugin listener : GaswConfiguration.getInstance().getListenerPlugins()) { + for (ListenerPlugin listener : listenerPlugins) { listener.jobSubmitted(job); } @@ -88,19 +86,13 @@ protected synchronized void add(Job job) throws GaswException { } } - /** - * Adds a job to be monitored. It should constructs a Job object and invoke - * the protected method add(job). - */ - public abstract void add(String jobID, String symbolicName, String fileName, - String parameters) throws GaswException; /** * Updates the job status and notifies listeners. */ protected void updateStatus(Job job) throws GaswException, DAOException { - for (ListenerPlugin listener : GaswConfiguration.getInstance().getListenerPlugins()) { + for (ListenerPlugin listener : listenerPlugins) { listener.jobStatusChanged(job); } jobDAO.update(job); @@ -148,6 +140,13 @@ protected boolean isReplica(Job job) throws DAOException { return jobDAO.getNumberOfCompletedJobsByInvocationID(job.getInvocationID()) > 0; } + public abstract void start(); + public abstract void terminate(); + /** + * Adds a job to be monitored. It should constructs a Job object and invoke + * the public method add(job). + */ + public abstract void add(String jobID, String symbolicName, String fileName, String parameters) throws GaswException; protected abstract void kill(Job job); protected abstract void reschedule(Job job); protected abstract void replicate(Job job); diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswOutputParser.java b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswOutputParser.java index ae92d505..9a1800e6 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswOutputParser.java +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswOutputParser.java @@ -33,75 +33,55 @@ package fr.insalyon.creatis.gasw.execution; import fr.insalyon.creatis.gasw.*; -import fr.insalyon.creatis.gasw.bean.*; -import fr.insalyon.creatis.gasw.dao.DAOException; -import fr.insalyon.creatis.gasw.dao.DAOFactory; +import fr.insalyon.creatis.gasw.bean.Data; +import fr.insalyon.creatis.gasw.bean.JobMinorStatus; +import fr.insalyon.creatis.gasw.bean.Node; +import fr.insalyon.creatis.gasw.bean.NodeID; +import fr.insalyon.creatis.gasw.dao.*; import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; -import java.io.*; -import java.net.URI; -import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class GaswOutputParser extends Thread { - - private static final Logger logger = LoggerFactory.getLogger(GaswOutputParser.class); - protected Job job; - protected File appStdOut; - protected File appStdErr; - protected BufferedWriter appStdOutWriter; - protected BufferedWriter appStdErrWriter; - protected List dataList; - protected Map uploadedResults; - protected StringBuilder inputsDownloadErrBuf; - protected StringBuilder resultsUploadErrBuf; - protected StringBuilder appStdOutBuf; - protected StringBuilder appStdErrBuf; - - public GaswOutputParser(String jobID) { - try { - this.job = DAOFactory.getDAOFactory().getJobDAO().getJobByID(jobID); - - this.appStdOut = getAppStdFile(GaswConstants.OUT_APP_EXT, GaswConstants.OUT_ROOT); - this.appStdErr = getAppStdFile(GaswConstants.ERR_APP_EXT, GaswConstants.ERR_ROOT); - - this.appStdOutWriter = new BufferedWriter(new FileWriter(appStdOut)); - this.appStdErrWriter = new BufferedWriter(new FileWriter(appStdErr)); - - this.inputsDownloadErrBuf = new StringBuilder(); - this.resultsUploadErrBuf = new StringBuilder(); - this.appStdOutBuf = new StringBuilder(); - this.appStdErrBuf = new StringBuilder(); - - this.dataList = new ArrayList(); - this.uploadedResults = null; - - } catch (IOException | DAOException ex) { - closeBuffers(); - logger.error("Error creating std out/err " + - "files and buffers for {}", jobID, ex); - } - } - - private void closeBuffers() { - try { - if (appStdOutWriter != null) { - appStdOutWriter.close(); - } - if (appStdErrWriter != null) { - appStdErrWriter.close(); - } - } catch (IOException ex) { - logger.error("Error closing buffers", ex); - } +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Scanner; + +public abstract class GaswOutputParser { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected final GaswConfiguration config; + protected final GaswNotification gaswNotification; + protected final JobDAO jobDAO; + protected final JobMinorStatusDAO jobMinorStatusDAO; + protected final NodeDAO nodeDAO; + protected final DataDAO dataDAO; + protected final List listenerPlugins; + + public GaswOutputParser(GaswConfiguration config, GaswNotification gaswNotification, + JobDAO jobDAO, JobMinorStatusDAO jobMinorStatusDAO, + NodeDAO nodeDAO, DataDAO dataDAO, List listenerPlugins) { + this.config = config; + this.gaswNotification = gaswNotification; + this.jobDAO = jobDAO; + this.jobMinorStatusDAO = jobMinorStatusDAO; + this.nodeDAO = nodeDAO; + this.dataDAO = dataDAO; + this.listenerPlugins = listenerPlugins; } - @Override - public void run() { + public void run(GaswParsingContext context) { try { - GaswOutput gaswOutput = getGaswOutput(); + GaswOutput gaswOutput = getGaswOutput(context); - for (ListenerPlugin listener : GaswConfiguration.getInstance().getListenerPlugins()) { + for (ListenerPlugin listener : listenerPlugins) { try { listener.jobFinished(gaswOutput); } catch (Exception ex) { @@ -113,34 +93,38 @@ public void run() { // remove this flag if it is not replicated after all try { // do not resubmit a job that was deliberately cancelled/killed - if (gaswOutput.getExitCode() == GaswExitCode.SUCCESS || gaswOutput.getExitCode() == GaswExitCode.EXECUTION_CANCELED || job.isBeingKilled()) { - job.setReplicating(false); - DAOFactory.getDAOFactory().getJobDAO().update(job); + if (gaswOutput.getExitCode() == GaswExitCode.SUCCESS + || gaswOutput.getExitCode() == GaswExitCode.EXECUTION_CANCELED + || context.getJob().isBeingKilled()) { + context.getJob().setReplicating(false); + jobDAO.update(context.getJob()); } else { - int retries = DAOFactory.getDAOFactory().getJobDAO().getFailedJobsByInvocationID(job.getInvocationID()).size() - 1; - if (retries < GaswConfiguration.getInstance().getDefaultRetryCount()) { - logger.warn("Job [{}] finished as \"{}\" (retried {} times).", job.getId(), job.getStatus().name(), retries); + // Processing error job here to avoid error throwing before it's labeled as error + gaswNotification.addErrorJob(gaswOutput); + int retries = jobDAO.getFailedJobsByInvocationID(context.getJob().getInvocationID()).size() - 1; + if (retries < config.getDefaultRetryCount()) { + logger.warn("Job [{}] finished as \"{}\" (retried {} times).", + context.getJob().getId(), context.getJob().getStatus().name(), retries); resubmit(); } else { - logger.warn("Job [{}] finished as \"{}\": holding job (max retries reached).", job.getId(), job.getStatus().name()); - if (job.getStatus() == GaswStatus.ERROR) { - job.setStatus(GaswStatus.ERROR_HELD); - } else if (job.getStatus() == GaswStatus.STALLED) { - job.setStatus(GaswStatus.STALLED_HELD); + logger.warn("Job [{}] finished as \"{}\": holding job (max retries reached).", + context.getJob().getId(), context.getJob().getStatus().name()); + if (context.getJob().getStatus() == GaswStatus.ERROR) { + context.getJob().setStatus(GaswStatus.ERROR_HELD); + } else if (context.getJob().getStatus() == GaswStatus.STALLED) { + context.getJob().setStatus(GaswStatus.STALLED_HELD); } - job.setReplicating(false); - DAOFactory.getDAOFactory().getJobDAO().update(job); + context.getJob().setReplicating(false); + jobDAO.update(context.getJob()); } - GaswNotification.getInstance().addErrorJob(gaswOutput); return; } } catch (DAOException | GaswException ex) { - logger.error("Error finalising job {}", job.getId(), ex); + logger.error("Error finalising job {}", context.getJob().getId(), ex); } - GaswNotification.getInstance().addFinishedJob(gaswOutput); - + gaswNotification.addFinishedJob(gaswOutput); } catch (GaswException ex) { - logger.error("Error processing output for job {}", job.getId(), ex); + logger.error("Error processing output for job {}", context.getJob().getId(), ex); } } @@ -151,23 +135,19 @@ public void run() { * respectively. * @throws GaswException */ - public abstract GaswOutput getGaswOutput() throws GaswException; + public abstract GaswOutput getGaswOutput(GaswParsingContext context) throws GaswException; - protected abstract void resubmit() throws GaswException; + protected void resubmit() throws GaswException {} - /** - * We use synchronized keyword in case of multiples jobs ending together (at the same time), - * it cause an issue if hibernate try to merge/add the same job inside the db - */ - protected int parseStdOut(File stdOut) { + protected int parseStdOut(File stdOut, GaswParsingContext context) throws IOException { int exitCode = -1; try { - if (job.getQueued() == null) { - job.setQueued(job.getCreation()); + if (context.getJob().getQueued() == null) { + context.getJob().setQueued(context.getJob().getCreation()); } - if (job.getDownload() == null) { - job.setDownload(job.getQueued()); + if (context.getJob().getDownload() == null) { + context.getJob().setDownload(context.getJob().getQueued()); } Node node = new Node(); @@ -189,33 +169,33 @@ protected int parseStdOut(File stdOut) { if (line.contains("")) { isAppExec = true; } else if (line.contains("")) { - isAppExec = false;; + isAppExec = false; } else if (isAppExec) { - appStdOutWriter.write(line + "\n"); - appStdOutBuf.append(line).append("\n"); + context.getAppStdOutWriter().write(line + "\n"); + context.getAppStdOutBuf().append(line).append("\n"); } // General Output if (line.contains("Input download time:")) { int downloadTime = Integer.parseInt(lineSplitted[lineSplitted.length - 2]); - job.setRunning(addDate(job.getDownload(), Calendar.SECOND, downloadTime)); + context.getJob().setRunning(addDate(context.getJob().getDownload(), Calendar.SECOND, downloadTime)); } else if (line.contains("Execution time:")) { - if (job.getRunning() == null) { - job.setRunning(job.getDownload()); + if (context.getJob().getRunning() == null) { + context.getJob().setRunning(context.getJob().getDownload()); } int executionTime = Integer.parseInt(lineSplitted[lineSplitted.length - 2]); - job.setUpload(addDate(job.getRunning(), Calendar.SECOND, executionTime)); + context.getJob().setUpload(addDate(context.getJob().getRunning(), Calendar.SECOND, executionTime)); } else if (line.contains("Results upload time:")) { int uploadTime = Integer.parseInt(lineSplitted[lineSplitted.length - 2]); - job.setEnd(addDate(job.getUpload(), Calendar.SECOND, uploadTime)); + context.getJob().setEnd(addDate(context.getJob().getUpload(), Calendar.SECOND, uploadTime)); } else if (line.contains("Exiting with return value")) { String[] errmsg = line.split("\\s+"); exitCode = Integer.parseInt(errmsg[errmsg.length - 1]); - job.setExitCode(exitCode); + context.getJob().setExitCode(exitCode); } else if (line.startsWith("===== uname =====")) { line = scanner.nextLine(); @@ -267,12 +247,11 @@ protected int parseStdOut(File stdOut) { } else if (line.startsWith("")) { isResultUpload = true; - uploadedResults = new HashMap(); } else if (line.startsWith("")) { isResultUpload = false; @@ -290,12 +269,12 @@ protected int parseStdOut(File stdOut) { uri = new URI(uploadedFile); } else { uri = lfcHost.isEmpty() - ? new URI("file://" + uploadedFile) - : new URI("lfn://" + lfcHost + uploadedFile); + ? new URI("file://" + uploadedFile) + : new URI("lfn://" + lfcHost + uploadedFile); } - uploadedResults.put(outputId, uri); - dataList.add(new Data(uri.toString(), Data.Type.Output)); - logger.info("Adding output {} {} for job {}" + outputId, uri, job.getId()); + context.putUploadedResult(outputId, uri); + context.addData(new Data(uri.toString(), Data.Type.Output)); + logger.info("Adding output {} {} for job {}", outputId, uri, context.getJob().getId()); } } } catch (Exception ex) { @@ -303,34 +282,38 @@ protected int parseStdOut(File stdOut) { } finally { scanner.close(); } - appStdOutWriter.close(); + context.getAppStdOutWriter().close(); - DAOFactory factory = DAOFactory.getDAOFactory(); if (nodeID.getSiteName() != null && nodeID.getNodeName() != null) { node.setNodeID(nodeID); - factory.getNodeDAO().add(node); - job.setNode(node); + nodeDAO.add(node); + context.getJob().setNode(node); } // Parse checkpoint - parseCheckpoint(); + parseCheckpoint(context); + + // Upsert Data rows before attaching them to the Job, to avoid unique constraint violations + for (Data d : context.getDataList()) { + dataDAO.upsertData(d); + } // Update Job - job.setData(dataList); - if (job.getEnd() == null) { - job.setEnd(new Date()); + context.getJob().setData(context.getDataList()); + if (context.getJob().getEnd() == null) { + context.getJob().setEnd(new Date()); } - factory.getJobDAO().update(job); + jobDAO.update(context.getJob()); } catch (DAOException | IOException ex) { - closeBuffers(); + context.closeBuffers(); logger.error("Error parsing stdout {}", stdOut.getAbsolutePath(), ex); } return exitCode; } - protected int parseStdErr(File stdErr, int exitCode) { + protected int parseStdErr(File stdErr, int exitCode, GaswParsingContext context) throws IOException { try { Scanner scanner = new Scanner(new FileInputStream(stdErr)); @@ -370,70 +353,66 @@ protected int parseStdErr(File stdErr, int exitCode) { isUploadTest = false; } else if (isAppExec) { - appStdErrWriter.write(line + "\n"); - appStdErrBuf.append(line).append("\n"); + context.getAppStdErrWriter().write(line + "\n"); + context.getAppStdErrBuf().append(line).append("\n"); } else if (isInputsDownload) { - inputsDownloadErrBuf.append(line).append("\n"); + context.getInputsDownloadErrBuf().append(line).append("\n"); } else if (isResultsUpload) { - resultsUploadErrBuf.append(line).append("\n"); + context.getResultsUploadErrBuf().append(line).append("\n"); } else if (isUploadTest) { - resultsUploadErrBuf.append(line).append("\n"); + context.getResultsUploadErrBuf().append(line).append("\n"); } if (line.contains("Exiting with return value")) { String[] errmsg = line.split("\\s+"); exitCode = Integer.valueOf(errmsg[errmsg.length - 1]).intValue(); - job.setExitCode(exitCode); + context.getJob().setExitCode(exitCode); } } } finally { scanner.close(); } - appStdErrWriter.close(); - DAOFactory.getDAOFactory().getJobDAO().update(job); + context.getAppStdErrWriter().close(); + jobDAO.update(context.getJob()); } catch (DAOException | IOException ex) { - closeBuffers(); + context.closeBuffers(); logger.error("Error parsing stderr {}", stdErr.getAbsolutePath(), ex); - } return exitCode; } - protected void parseNonStdOut(int exitCode) { + protected void parseNonStdOut(int exitCode, GaswParsingContext context) throws IOException { try { - job.setEnd(new Date()); - DAOFactory factory = DAOFactory.getDAOFactory(); + context.getJob().setEnd(new Date()); - for (JobMinorStatus minorStatus : factory.getJobMinorStatusDAO().getExecutionMinorStatus(job.getId())) { + for (JobMinorStatus minorStatus : jobMinorStatusDAO.getExecutionMinorStatus(context.getJob().getId())) { switch (minorStatus.getStatus()) { case Application: - job.setRunning(minorStatus.getDate()); + context.getJob().setRunning(minorStatus.getDate()); break; case Outputs: - job.setUpload(minorStatus.getDate()); + context.getJob().setUpload(minorStatus.getDate()); } } - parseCheckpoint(); - job.setExitCode(exitCode); - factory.getJobDAO().update(job); + parseCheckpoint(context); + context.getJob().setExitCode(exitCode); + jobDAO.update(context.getJob()); } catch (DAOException ex) { - closeBuffers(); + context.closeBuffers(); logger.error("Error parsing NonStdOut", ex); } } - private void parseCheckpoint() { + private void parseCheckpoint(GaswParsingContext context) throws IOException { try { - DAOFactory factory = DAOFactory.getDAOFactory(); - - List list = factory.getJobMinorStatusDAO().getCheckpoints(job.getId()); + List list = jobMinorStatusDAO.getCheckpoints(context.getJob().getId()); if (!list.isEmpty()) { int sumCheckpointInit = 0; @@ -461,44 +440,29 @@ private void parseCheckpoint() { } } - job.setCheckpointInit(sumCheckpointInit); - job.setCheckpointUpload(sumCheckpointUpload); + context.getJob().setCheckpointInit(sumCheckpointInit); + context.getJob().setCheckpointUpload(sumCheckpointUpload); } } catch (DAOException ex) { - closeBuffers(); + context.closeBuffers(); logger.error("Error parsing checkpoints", ex); } } - protected File saveFile(String extension, String dir, String content) { - FileWriter fstream = null; + protected File saveFile(String extension, String dir, String content, GaswParsingContext context) { + Path path = Path.of(dir, context.getJob().getFileName() + ".sh" + extension); try { - File stdDir = new File(dir); - if (!stdDir.exists()) { - stdDir.mkdir(); - } - File stdFile = new File(dir + "/" + job.getFileName() + ".sh" + extension); - fstream = new FileWriter(stdFile); - BufferedWriter out = new BufferedWriter(fstream); - out.write(content); - out.close(); - - return stdFile; - + Files.createDirectories(path.getParent()); + Files.writeString(path, content); + return path.toFile(); } catch (IOException ex) { - logger.error("Error:", ex); - } finally { - try { - fstream.close(); - } catch (IOException ex) { - logger.error("Error:", ex); - } + logger.error("Error writing file {}", path, ex); + return null; } - return null; } - protected File moveAppFile(File source, String extension, String dir) { - File dest = getAppStdFile(extension, dir); + protected File moveAppFile(File source, String extension, String dir, GaswParsingContext context) { + File dest = context.getAppStdFile(extension, dir); if (source.exists()) { source.renameTo(dest); } else { @@ -507,41 +471,13 @@ protected File moveAppFile(File source, String extension, String dir) { return dest; } - protected File moveProvenanceFile(String sourceDir) { - String provenanceFileName = getAppStdFileName(GaswConstants.PROVENANCE_EXT); + protected File moveProvenanceFile(String sourceDir, GaswParsingContext context) { + String provenanceFileName = context.getAppStdFileName(GaswConstants.PROVENANCE_EXT); return moveAppFile( new File(sourceDir, provenanceFileName), GaswConstants.PROVENANCE_EXT, - GaswConstants.PROVENANCE_ROOT); - } - - protected File getAppStdFile(String extension, String dir) { - File stdDir = new File(dir); - - if (!stdDir.exists()) { - stdDir.mkdirs(); - } - return new File(dir + "/" + getAppStdFileName(extension)); - } - - protected String getAppStdFileName(String extension) { - return job.getFileName() + ".sh" + extension; - } - - protected String getInputsDownloadErr() { - return inputsDownloadErrBuf.toString(); - } - - protected String getResultsUploadErr() { - return resultsUploadErrBuf.toString(); - } - - protected String getAppStdErr() { - return appStdErrBuf.toString(); - } - - protected String getAppStdOut() { - return appStdOutBuf.toString(); + GaswConstants.PROVENANCE_ROOT, + context); } private Date addDate(Date dateToBeAdded, int field, int amount) { diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswParsingContext.java b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswParsingContext.java new file mode 100644 index 00000000..890b4d84 --- /dev/null +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswParsingContext.java @@ -0,0 +1,150 @@ +package fr.insalyon.creatis.gasw.execution; + +import fr.insalyon.creatis.gasw.GaswConstants; +import fr.insalyon.creatis.gasw.bean.Data; +import fr.insalyon.creatis.gasw.bean.Job; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// Job-specific context used by GaswOutputParser +public class GaswParsingContext { + + private final Job job; + private final File appStdOutFile; + private final File appStdErrFile; + private final BufferedWriter appStdOutWriter; + private final BufferedWriter appStdErrWriter; + private final List dataList; + private final Map uploadedResults; + private final StringBuilder inputsDownloadErrBuf; + private final StringBuilder resultsUploadErrBuf; + private final StringBuilder appStdOutBuf; + private final StringBuilder appStdErrBuf; + + public GaswParsingContext(Job job) throws IOException { + try { + this.job = job; + + this.appStdOutFile = getAppStdFile(GaswConstants.OUT_APP_EXT, GaswConstants.OUT_ROOT); + this.appStdErrFile = getAppStdFile(GaswConstants.ERR_APP_EXT, GaswConstants.ERR_ROOT); + + appStdOutWriter = new BufferedWriter(new FileWriter(this.appStdOutFile)); + appStdErrWriter = new BufferedWriter(new FileWriter(this.appStdErrFile)); + + inputsDownloadErrBuf = new StringBuilder(); + resultsUploadErrBuf = new StringBuilder(); + appStdOutBuf = new StringBuilder(); + appStdErrBuf = new StringBuilder(); + + dataList = new ArrayList<>(); + uploadedResults = new HashMap<>(); + + } catch (IOException e) { + closeBuffers(); + throw new IOException("Error creating std out/err files and buffers for job " + job.getId(), e); + } + } + + public void closeBuffers() throws IOException { + try { + if (appStdOutWriter != null) { + appStdOutWriter.close(); + } + if (appStdErrWriter != null) { + appStdErrWriter.close(); + } + } catch (IOException e) { + throw new IOException("Error closing buffers", e); + } + } + + public File getAppStdOutFile() { + return appStdOutFile; + } + + public File getAppStdErrFile() { + return appStdErrFile; + } + + public File getAppStdFile(String extension, String dir) { + File stdDir = new File(dir); + + if (!stdDir.exists()) { + stdDir.mkdirs(); + } + + return new File(dir + "/" + getAppStdFileName(extension)); + } + + public String getAppStdFileName(String extension) { + return job.getFileName() + ".sh" + extension; + } + + public String getInputsDownloadErr() { + return inputsDownloadErrBuf.toString(); + } + + public String getResultsUploadErr() { + return resultsUploadErrBuf.toString(); + } + + public String getAppStdErr() { + return appStdErrBuf.toString(); + } + + public String getAppStdOut() { + return appStdOutBuf.toString(); + } + + public Job getJob() { + return job; + } + + public BufferedWriter getAppStdOutWriter() { + return appStdOutWriter; + } + + public BufferedWriter getAppStdErrWriter() { + return appStdErrWriter; + } + + public StringBuilder getAppStdOutBuf() { + return appStdOutBuf; + } + + public StringBuilder getAppStdErrBuf() { + return appStdErrBuf; + } + + public StringBuilder getInputsDownloadErrBuf() { + return inputsDownloadErrBuf; + } + + public StringBuilder getResultsUploadErrBuf() { + return resultsUploadErrBuf; + } + + public List getDataList() { + return dataList; + } + + public void addData(Data data) { + dataList.add(data); + } + + public Map getUploadedResults() { + return uploadedResults; + } + + public void putUploadedResult(String id, URI uri) { + uploadedResults.put(id, uri); + } +} \ No newline at end of file diff --git a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswSubmit.java b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswSubmit.java index 3f237722..fe8ceb7d 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/execution/GaswSubmit.java +++ b/src/main/java/fr/insalyon/creatis/gasw/execution/GaswSubmit.java @@ -32,9 +32,7 @@ */ package fr.insalyon.creatis.gasw.execution; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -54,27 +52,31 @@ public abstract class GaswSubmit { - private static final Logger logger = LoggerFactory.getLogger(GaswSubmit.class); - protected GaswInput gaswInput; + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final GaswConfiguration config; + private final FailOver failOver; + private final MoteurliteConfigGenerator moteurliteConfigGenerator; + protected String scriptName; protected String jdlName; - protected GaswMinorStatusServiceGenerator minorStatusServiceGenerator; - public GaswSubmit(GaswInput gaswInput, GaswMinorStatusServiceGenerator minorStatusServiceGenerator) - throws GaswException { - - this.gaswInput = gaswInput; - this.minorStatusServiceGenerator = minorStatusServiceGenerator; + public GaswSubmit(GaswConfiguration config, FailOver failOver, MoteurliteConfigGenerator moteurliteConfigGenerator) { + this.config = config; + this.failOver = failOver; + this.moteurliteConfigGenerator = moteurliteConfigGenerator; + } - if (GaswConfiguration.getInstance().isFailOverEnabled()) { - FailOver.getInstance().addData(gaswInput.getDownloads()); + public String submit(GaswInput gaswInput, GaswMonitor gaswMonitor) { + if (this.config.isFailOverEnabled()) { + this.failOver.addData(gaswInput.getDownloads()); } - } - public abstract String submit() throws GaswException; + return scriptName; + } - protected String generateScript() throws GaswException { + protected String generateScript(GaswInput gaswInput) throws GaswException { try { String scriptName; @@ -82,14 +84,14 @@ protected String generateScript() throws GaswException { logger.info("MoteurLite is enabled, generating Moteurlite-specific script."); // Generate the Moteurlite-specific configuration - Map configMoteurlite = MoteurliteConfigGenerator.getInstance().generateConfig(gaswInput, minorStatusServiceGenerator); + Map configMoteurlite = moteurliteConfigGenerator.generateConfig(gaswInput); // Publish the configuration and invocation publishConfiguration(gaswInput.getJobId(), configMoteurlite); publishInvocation(gaswInput.getJobId(), gaswInput.getInvocationString()); // Publish the script itself - scriptName = publishMoteurLiteScript(); + scriptName = publishMoteurLiteScript(gaswInput.getJobId()); return scriptName; @@ -99,17 +101,16 @@ protected String generateScript() throws GaswException { } } - private String publishMoteurLiteScript() throws IOException, GaswException { + private String publishMoteurLiteScript(String jobId) throws IOException, GaswException { prepareScriptDir(); try { // If MoteurLite is enabled, use the jobId as the script name - String fileName = gaswInput.getJobId(); - Path destScriptFile = Paths.get(GaswConstants.SCRIPT_ROOT, fileName); + Path destScriptFile = Paths.get(GaswConstants.SCRIPT_ROOT, jobId); try (InputStream is = getClass().getClassLoader().getResourceAsStream("script.sh")) { Files.copy(is, destScriptFile, StandardCopyOption.REPLACE_EXISTING); } - return fileName; + return jobId; } catch (Exception e) { logger.error("Error getting script file from classpath", e); throw new GaswException(e); @@ -149,11 +150,7 @@ protected String publishJdl(String scriptName, String jdl) { * @throws IOException */ private void writeToFile(String filePath, String contents) throws IOException { - FileWriter fstream = new FileWriter(filePath); - BufferedWriter out = new BufferedWriter(fstream); - out.write(contents); - out.close(); - fstream.close(); + Files.writeString(Path.of(filePath), contents); } private void publishConfiguration(String jobId, Map config) throws IOException { diff --git a/src/main/java/fr/insalyon/creatis/gasw/plugin/DatabasePlugin.java b/src/main/java/fr/insalyon/creatis/gasw/plugin/DatabasePlugin.java index 91fcff7b..ac252962 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/plugin/DatabasePlugin.java +++ b/src/main/java/fr/insalyon/creatis/gasw/plugin/DatabasePlugin.java @@ -35,13 +35,12 @@ package fr.insalyon.creatis.gasw.plugin; import fr.insalyon.creatis.gasw.GaswException; -import net.xeoh.plugins.base.Plugin; /** * * @author Rafael Silva */ -public interface DatabasePlugin extends Plugin { +public interface DatabasePlugin { /** * Gets the name of the plugin. @@ -50,22 +49,13 @@ public interface DatabasePlugin extends Plugin { */ public String getName(); - /** - * This is the first method invoked by GASW. This method is called when GASW - * is loading its configuration. It is useful to load plugin properties from - * the configuration file. - * - * @throws GaswException - */ - public void load() throws GaswException; - /** * Gets the database schema name. * * @return * @throws GaswException */ - public String getSchema() throws GaswException; + public String getSchema(); /** * Gets the JDBC driver. @@ -73,7 +63,7 @@ public interface DatabasePlugin extends Plugin { * @return * @throws GaswException */ - public String getDriverClass() throws GaswException; + public String getDriverClass(); /** * Gets the JDBC connection URL. @@ -81,7 +71,7 @@ public interface DatabasePlugin extends Plugin { * @return * @throws GaswException */ - public String getConnectionUrl() throws GaswException; + public String getConnectionUrl(); /** * Gets the hibernate dialect. @@ -91,7 +81,7 @@ public interface DatabasePlugin extends Plugin { * @return * @throws GaswException */ - public String getHibernateDialect() throws GaswException; + public String getHibernateDialect(); /** * Gets the database username. @@ -99,7 +89,7 @@ public interface DatabasePlugin extends Plugin { * @return * @throws GaswException */ - public String getUserName() throws GaswException; + public String getUserName(); /** * Gets the database password. @@ -107,5 +97,5 @@ public interface DatabasePlugin extends Plugin { * @return * @throws GaswException */ - public String getPassword() throws GaswException; + public String getPassword(); } diff --git a/src/main/java/fr/insalyon/creatis/gasw/plugin/ExecutorPlugin.java b/src/main/java/fr/insalyon/creatis/gasw/plugin/ExecutorPlugin.java index a7f3fe88..f0cca7d4 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/plugin/ExecutorPlugin.java +++ b/src/main/java/fr/insalyon/creatis/gasw/plugin/ExecutorPlugin.java @@ -34,14 +34,12 @@ import fr.insalyon.creatis.gasw.GaswException; import fr.insalyon.creatis.gasw.GaswInput; -import java.util.List; -import net.xeoh.plugins.base.Plugin; /** * * @author Rafael Ferreira da Silva */ -public interface ExecutorPlugin extends Plugin { +public interface ExecutorPlugin { /** * Gets the executor name. @@ -51,28 +49,13 @@ public interface ExecutorPlugin extends Plugin { public String getName(); /** - * Prepares the executor to submit a job with the specified inputs. + * Submits the job with the specified inputs. * * @param gaswInput Job inputs - * @throws GaswException - */ - public void load(GaswInput gaswInput) throws GaswException; - - /** - * Gets a list of persistent classes to be loaded in Hibernate. - * - * @return List of persistent classes - * @throws GaswException - */ - public List getPersistentClasses() throws GaswException; - - /** - * Submits the job. - * * @return Job identification * @throws GaswException */ - public String submit() throws GaswException; + public String submit(GaswInput gaswInput) throws GaswException; /** * Finalizes the executor. diff --git a/src/main/java/fr/insalyon/creatis/gasw/plugin/ListenerPlugin.java b/src/main/java/fr/insalyon/creatis/gasw/plugin/ListenerPlugin.java index 155a0671..f0001e63 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/plugin/ListenerPlugin.java +++ b/src/main/java/fr/insalyon/creatis/gasw/plugin/ListenerPlugin.java @@ -34,38 +34,24 @@ */ package fr.insalyon.creatis.gasw.plugin; -import fr.insalyon.creatis.gasw.GaswException; import fr.insalyon.creatis.gasw.GaswOutput; import fr.insalyon.creatis.gasw.bean.Job; import fr.insalyon.creatis.gasw.bean.JobMinorStatus; -import java.util.List; -import net.xeoh.plugins.base.Plugin; - /** * * @author Rafael Silva */ -public interface ListenerPlugin extends Plugin { - - public String getPluginName(); +public interface ListenerPlugin { - /** - * Gets a list of persistent classes to be loaded in Hibernate. - * - * @return List of persistent classes - * @throws GaswException - */ - public List getPersistentClasses() throws GaswException; - - public void load() throws GaswException; + public String getName(); - public void jobSubmitted(Job job) throws GaswException; + public void jobSubmitted(Job job); - public void jobFinished(GaswOutput gaswOutput) throws GaswException; + public void jobFinished(GaswOutput gaswOutput); - public void jobStatusChanged(Job job) throws GaswException; + public void jobStatusChanged(Job job); - public void jobMinorStatusReported(JobMinorStatus jobMinorStatus) throws GaswException; + public void jobMinorStatusReported(JobMinorStatus jobMinorStatus); - public void terminate() throws GaswException; + public void terminate(); } diff --git a/src/main/java/fr/insalyon/creatis/gasw/script/MoteurliteConfigGenerator.java b/src/main/java/fr/insalyon/creatis/gasw/script/MoteurliteConfigGenerator.java index d59f9d77..b4588cd4 100644 --- a/src/main/java/fr/insalyon/creatis/gasw/script/MoteurliteConfigGenerator.java +++ b/src/main/java/fr/insalyon/creatis/gasw/script/MoteurliteConfigGenerator.java @@ -43,26 +43,20 @@ import fr.insalyon.creatis.gasw.GaswConstants; import fr.insalyon.creatis.gasw.GaswException; import fr.insalyon.creatis.gasw.GaswInput; -import fr.insalyon.creatis.gasw.execution.GaswMinorStatusServiceGenerator; -public class MoteurliteConfigGenerator { +import org.springframework.stereotype.Service; - private static MoteurliteConfigGenerator instance; - private GaswConfiguration conf; +@Service +public class MoteurliteConfigGenerator { - public synchronized static MoteurliteConfigGenerator getInstance() throws GaswException { - if (instance == null) { - instance = new MoteurliteConfigGenerator(); - } - return instance; - } + private final GaswConfiguration conf; - private MoteurliteConfigGenerator() throws GaswException { - conf = GaswConfiguration.getInstance(); + public MoteurliteConfigGenerator(GaswConfiguration config) throws GaswException { + conf = config; } // Generates the configuration based on the input and minor status service - public Map generateConfig(GaswInput gaswInput, GaswMinorStatusServiceGenerator minorStatusService) + public Map generateConfig(GaswInput gaswInput) throws IOException { Map config = new HashMap<>(); if (gaswInput.getExecutableName() != null) { diff --git a/src/main/java/fr/insalyon/creatis/gasw/util/VelocityUtil.java b/src/main/java/fr/insalyon/creatis/gasw/util/VelocityUtil.java deleted file mode 100644 index d9003b70..00000000 --- a/src/main/java/fr/insalyon/creatis/gasw/util/VelocityUtil.java +++ /dev/null @@ -1,136 +0,0 @@ -/* Copyright CNRS-CREATIS - * - * Rafael Ferreira da Silva - * rafael.silva@creatis.insa-lyon.fr - * http://www.rafaelsilva.com - * - * This software is a grid-enabled data-driven workflow manager and editor. - * - * This software is governed by the CeCILL license under French law and - * abiding by the rules of distribution of free software. You can use, - * modify and/ or redistribute the software under the terms of the CeCILL - * license as circulated by CEA, CNRS and INRIA at the following URL - * "http://www.cecill.info". - * - * As a counterpart to the access to the source code and rights to copy, - * modify and redistribute granted by the license, users are provided only - * with a limited warranty and the software's author, the holder of the - * economic rights, and the successive licensors have only limited - * liability. - * - * In this respect, the user's attention is drawn to the risks associated - * with loading, using, modifying and/or developing or reproducing the - * software by the user in light of its specific status of free software, - * that may mean that it is complicated to manipulate, and that also - * therefore means that it is reserved for developers and experienced - * professionals having in-depth computer knowledge. Users are therefore - * encouraged to load and test the software's suitability as regards their - * requirements in conditions enabling the security of their systems and/or - * data to be ensured and, more generally, to use and operate it in the - * same conditions as regards security. - * - * The fact that you are presently reading this means that you have had - * knowledge of the CeCILL license and that you accept its terms. - */ -package fr.insalyon.creatis.gasw.util; - -import fr.insalyon.creatis.gasw.GaswException; - -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.util.Properties; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.velocity.Template; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.VelocityEngine; -import org.apache.velocity.exception.MethodInvocationException; -import org.apache.velocity.exception.ParseErrorException; -import org.apache.velocity.exception.ResourceNotFoundException; -import org.apache.velocity.runtime.log.NullLogChute; -import org.apache.velocity.runtime.resource.loader.StringResourceLoader; -import org.apache.velocity.runtime.resource.util.StringResourceRepository; - -public class VelocityUtil { - - private static final Logger logger = LoggerFactory.getLogger(VelocityUtil.class); - private static volatile VelocityEngine ve; - private Template template; - private VelocityContext context; - - public VelocityUtil(String templatePath) throws Exception { - this(templatePath, true); - } - - public VelocityUtil(String templatePath, boolean enableLogging) throws Exception { - if (ve == null) { - Properties properties = new Properties(); - properties.setProperty("resource.loader", "string"); - properties.setProperty("string.resource.loader.class", "org.apache.velocity.runtime.resource.loader.StringResourceLoader"); - properties.setProperty("string.resource.loader.repository.class", "org.apache.velocity.runtime.resource.util.StringResourceRepositoryImpl"); - - if ( ! enableLogging) { - properties.setProperty("runtime.log.logsystem.class", NullLogChute.class.getName()); - } - - ve = new VelocityEngine(properties); - ve.init(); - } - - if (!ve.resourceExists(templatePath)) { - StringResourceRepository repo = StringResourceLoader.getRepository(); - repo.putStringResource(templatePath, getTemplateFromResource(templatePath)); - } - - template = ve.getTemplate(templatePath); - context = new VelocityContext(); - } - - /** - * Adds data to the context. - * - * @param key - * @param value - */ - public void put(String key, Object value) { - - context.put(key, value); - } - - /** - * Renders the template into a StringWriter. - * - * @return - * @throws GaswException - */ - public StringWriter merge() throws GaswException { - try { - StringWriter writer = new StringWriter(); - template.merge(context, writer); - - return writer; - - } catch (ResourceNotFoundException ex) { - logger.error("Error:", ex); - throw new GaswException(ex); - } catch (ParseErrorException ex) { - logger.error("Error:", ex); - throw new GaswException(ex); - } catch (MethodInvocationException ex) { - logger.error("Error:", ex); - throw new GaswException(ex); - } - } - - private String getTemplateFromResource(final String templatePath) { - try { - InputStream stream = ClassLoader.getSystemResourceAsStream(templatePath); - return IOUtils.toString(stream, "UTF-8"); - - } catch (IOException ex) { - throw new RuntimeException(ex); - } - } -} diff --git a/src/main/resources/gasw.properties b/src/main/resources/gasw.properties new file mode 100644 index 00000000..56d0ff11 --- /dev/null +++ b/src/main/resources/gasw.properties @@ -0,0 +1,37 @@ +gasw.default.executor=Local +gasw.default.environment= +gasw.default.background-script= +gasw.default.requirements= +gasw.default.retry-count=5 +gasw.default.timeout=100000 +gasw.default.sleep-time=20 +gasw.default.cpu-time=8640 + +gasw.vo.name=biomed +gasw.vo.default-SE=SBG-disk +gasw.vo.use-close-SE=true + +gasw.boutiques.bosh-CVMFS-path=/cvmfs/biomed.egi.eu/vip/virtualenv/bin +gasw.boutiques.provenance-dir=${HOME}/.cache/boutiques/data +gasw.boutiques.file-name=workflow.json + +gasw.containers.singularity-path=/cvmfs/dirac.egi.eu/dirac/v8.0.39/Linux-x86_64/bin +gasw.containers.udocker-tag=1.3.1 +gasw.containers.runtime=docker +gasw.containers.images-base-path=/cvmfs/biomed.egi.eu/vip/singularity +gasw.containers.CVMFS-path=/cvmfs/biomed.egi.eu/vip/udocker/containers + +gasw.failover.enabled=false +gasw.failover.host=localhost +gasw.failover.port=8446 +gasw.failover.home=/dpm/localhost/generated +gasw.failover.max-retry=3 + +gasw.minor-status.enabled=false +gasw.download.min-avg-throughput=150 +gasw.source.script= + +gasw.scheduler.pool-size=10 +gasw.scheduler.thread-name-prefix=gasw-scheduler- +gasw.scheduler.wait-for-tasks-on-shutdown=true +gasw.scheduler.await-termination-seconds=60 diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 58acc3c3..f0aeccc9 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -44,4 +44,6 @@ + + diff --git a/src/main/resources/vm/script/basic/cleanupFunction.vm b/src/main/resources/vm/script/basic/cleanupFunction.vm deleted file mode 100644 index eb28736e..00000000 --- a/src/main/resources/vm/script/basic/cleanupFunction.vm +++ /dev/null @@ -1,38 +0,0 @@ -## cleanupFunction.vm - -## Variables -## $cacheDir, $cacheFile - -function cleanup { - if [[ $isGfalmountExec -eq 0 ]] #flag checks if directories are mounted with gfal - then - unmountGfal #unmounts all gfal mounted directories - unlink /tmp/*_$(basename $PWD) - fi - startLog cleanup - info "=== ls -a ===" - ls -a - info "=== ls $cacheDir/$cacheFile ===" - ls $cacheDir/$cacheFile - info "=== cat $cacheDir/$cacheFile === " - cat $cacheDir/$cacheFile - info "Cleaning up: rm * -Rf" - \rm * -Rf - if [ "${BACKPID}" != "" ] - then - for i in `ps --ppid ${BACKPID} -o pid | grep -v PID` - do - info "Killing child of background script (pid ${i})" - kill -9 ${i} - done - info "Killing background script (pid ${BACKPID})" - kill -9 ${BACKPID} - fi - info "END date:" - date +%s - stopLog cleanup - check_cleanup=true -} - -export -f cleanup -trap 'echo "trap activation" && stopRefreshingToken | if [ "$check_cleanup" = true ]; then echo "cleanup was already executed successfully"; else echo "Executing cleanup" && cleanup; fi' INT EXIT diff --git a/src/main/resources/vm/script/basic/logFunctions.vm b/src/main/resources/vm/script/basic/logFunctions.vm deleted file mode 100644 index db5cc8e0..00000000 --- a/src/main/resources/vm/script/basic/logFunctions.vm +++ /dev/null @@ -1,27 +0,0 @@ -## logFunctions.vm - -function info { - local D=`date` - echo [ INFO - $D ] $* -} - -function warning { - local D=`date` - echo [ WARN - $D ] $* -} - -function error { - local D=`date` - echo [ ERROR - $D ] $* >&2 -} - -function startLog { - echo "<$*>" >&1 - echo "<$*>" >&2 -} - -function stopLog { - local logName=$1 - echo "" >&1 - echo "" >&2 -} \ No newline at end of file diff --git a/src/main/resources/vm/script/datamanagement/addToCacheFunction.vm b/src/main/resources/vm/script/datamanagement/addToCacheFunction.vm deleted file mode 100644 index ddce1c6b..00000000 --- a/src/main/resources/vm/script/datamanagement/addToCacheFunction.vm +++ /dev/null @@ -1,36 +0,0 @@ -## addToCacheFunction.vm - -## Variables -## $cacheDir, $cacheFile - -function addToCache { - - mkdir -p $cacheDir - touch $cacheDir/$cacheFile - local LFN=$1 - local FILE=`basename $2` - local i=0 - local exist="true" - local NAME="" - while [ "${exist}" = "true" ] - do - NAME="$cacheDir/${FILE}-cache-${i}" - test -f ${NAME} - if [ $? != 0 ] - then - exist="false" - fi - i=`expr $i + 1` - done - info "Removing all cache entries for ${LFN} (files will stay locally in case anyone else needs them)" - local TEMP=`mktemp temp.XXXXXX` - awk -v L=${LFN} '$1!=L {print}' $cacheDir/$cacheFile > ${TEMP} - \mv -f ${TEMP} $cacheDir/$cacheFile - info "Adding file ${FILE} to cache and setting the timestamp" - \cp -f ${FILE} ${NAME} - local date_local=`ls -la ${NAME} | awk -F' ' '{print $6, $7, $8}'` - local TIMESTAMP=`date -d "${date_local}" +%s` - echo "${LFN} ${NAME} ${TIMESTAMP}" >> $cacheDir/$cacheFile -} - -export -f addToCache \ No newline at end of file diff --git a/src/main/resources/vm/script/datamanagement/addToFailOverFunction.vm b/src/main/resources/vm/script/datamanagement/addToFailOverFunction.vm deleted file mode 100644 index 4b4e5c36..00000000 --- a/src/main/resources/vm/script/datamanagement/addToFailOverFunction.vm +++ /dev/null @@ -1,42 +0,0 @@ -## addToFailOverFunction.vm - -## Variables -## $failOverHost, $failOverPort, $failOverHome - -function addToFailOver { - - local LFN=$1 - local FILE=$2 - local REMOTEFILE=`lcg-lr lfn:${LFN} | grep $failOverHost` -#set( $generated = '${REMOTEFILE#*generated}' ) - local RPFILE=${generated} - - lcg-del --nobdii --defaultsetype srmv2 -v srm://$failOverHost:$failOverPort/srm/managerv2?SFN=$failOverHome${RPFILE} &>/dev/null - lfc-ls ${LFN} - if [ $? = 0 ] - then - lfc-rename ${LFN} ${LFN}-garbage-`date +"%Y-%m-%d-%H-%M-%S"` - fi - lfc-mkdir -p `dirname ${LFN}`; - local FILENAME=`echo $RANDOM$RANDOM | md5sum | awk '{print $1}'` - local FOLDERNAME=`date +"%Y-%m-%d"` - local OPTS="--nobdii --defaultsetype srmv2" - DM_DEST="srm://$failOverHost:$failOverPort/srm/managerv2?SFN=$failOverHome/${FOLDERNAME}/file-${FILENAME}" - GUID=`lcg-cr ${OPTS} -d ${DM_DEST} file:${FILE}` - if [ $? = 0 ] - then - lcg-aa ${GUID} lfn:${LFN} - if [ $? = 0 ] - then - info "Data successfully copied to Fail Over." - else - error "Unable to create LFN alias ${LFN} to ${GUID}" - return 1 - fi - else - error "Unable to copy data to Fail Over." - return 1 - fi -} - -export -f addToFailOver \ No newline at end of file diff --git a/src/main/resources/vm/script/datamanagement/checkCacheDownloadAndCacheLFNFunction.vm b/src/main/resources/vm/script/datamanagement/checkCacheDownloadAndCacheLFNFunction.vm deleted file mode 100644 index bebe1ff8..00000000 --- a/src/main/resources/vm/script/datamanagement/checkCacheDownloadAndCacheLFNFunction.vm +++ /dev/null @@ -1,91 +0,0 @@ -## checkCacheDownloadAndCacheLFNFunction.vm - -## Variables -## $cacheDir, $cacheFile - -function checkCacheDownloadAndCacheLFN { - - local LFN=$1 - # the LFN is assumed to be in the /grid/biomed/... format (no leading lfn://lfc-biomed.in2p3.fr:5010/) - # this variable is true <=> the file has to be downloaded again - local download="true" - # first check if the file is already in cache - local LOCALPATH=`awk -v L=${LFN} '$1==L {print $2}' $cacheDir/$cacheFile` - if [ "${LOCALPATH}" != "" ] - then - info "There is an entry in the cache: test if the local file still here" - local TIMESTAMP_LOCAL="" - local TIMESTAMP_GRID="" - local date_local="" - test -f ${LOCALPATH} - if [ $? = 0 ] - then - info "The file exists: checking if it was modified since it was added to the cache" - local YEAR=`date +%Y` - local YEARBEFORE=`expr ${YEAR} - 1` - local currentDate=`date +%s` - local TIMESTAMP_CACHE=`awk -v L=${LFN} '$1==L {print $3}' $cacheDir/$cacheFile` - local LOCALMONTH=`ls -la ${LOCALPATH} | awk -F' ' '{print $6}'` - local MONTHTIME=`date -d "${LOCALMONTH} 1 00:00" +%s` - date_local=`ls -la ${LOCALPATH} | awk -F' ' '{print $6, $7, $8}'` - if [ "${MONTHTIME}" -gt "${currentDate}" ] - then - TIMESTAMP_LOCAL=`date -d "${date_local} ${YEARBEFORE}" +%s` - else - TIMESTAMP_LOCAL=`date -d "${date_local} ${YEAR}" +%s` - fi - if [ "${TIMESTAMP_CACHE}" = "${TIMESTAMP_LOCAL}" ] - then - info "The file was not touched since it was added to the cache: test if it is up up-to-date" - local date_grid_s=`lfc-ls -l ${LFN} | awk -F' ' '{print $6, $7, $8}'` - local MONTHGRID=`echo ${date_grid_s} | awk -F' ' '{print $1}'` - MONTHTIME=`date -d "${MONTHGRID} 1 00:00" +%s` - if [ "${MONTHTIME}" != "" ] && [ "${date_grid_s}" != "" ] - then - if [ "${MONTHTIME}" -gt "${currentDate}" ] - then - # it must be last year - TIMESTAMP_GRID=`date -d "${date_grid_s} ${YEARBEFORE}" +%s` - else - TIMESTAMP_GRID=`date -d "${date_grid_s} ${YEAR}" +%s` - fi - if [ "${TIMESTAMP_LOCAL}" -gt "${TIMESTAMP_GRID}" ] - then - info "The file is up-to-date ; there is no need to download it again" - download="false" - else - warning "The cache entry is outdated (local modification date is ${TIMESTAMP_LOCAL} - ${date_local} while grid is ${TIMESTAMP_GRID} ${date_grid_s})" - fi - else - warning "Cannot determine file timestamp on the LFC" - fi - else - warning "The cache entry was modified since it was created (cache time is ${TIMESTAMP_CACHE} and file time is ${TIMESTAMP_LOCAL} - ${date_local})" - fi - else - warning "The cache entry disappeared" - fi - else - info "There is no entry in the cache" - fi - if [ "${download}" = "false" ] - then - info "Linking file from cache: ${LOCALPATH}" - BASE=`basename ${LFN}` - info "ln -s ${LOCALPATH} ./${BASE}" - ln -s ${LOCALPATH} ./${BASE} - return 0 - fi - if [ "${download}" = "true" ] - then - downloadLFN ${LFN} - if [ $? != 0 ] - then - return 1 - fi - addToCache ${LFN} `basename ${LFN}` - return 0 - fi -} - -export -f checkCacheDownloadAndCacheLFN \ No newline at end of file diff --git a/src/main/resources/vm/script/datamanagement/deleteFunctions.vm b/src/main/resources/vm/script/datamanagement/deleteFunctions.vm deleted file mode 100644 index cbdb3336..00000000 --- a/src/main/resources/vm/script/datamanagement/deleteFunctions.vm +++ /dev/null @@ -1,38 +0,0 @@ -## deleteFunctions.vm - -## Unused variables: -## $failOverEnabled - -function delete { - - local URI=$1 - local TEST=$2 - - startLog file_delete uri="${URI}" - - ## The pattern must NOT be put between quotation marks. - if [[ ${URI} == girder:/* ]] - then - info "delete not supported for girder" - elif [[ ${URI} == file:/* ]] - then - local FILENAME=`echo $URI | sed 's%file://*%/%'` - - info "Removing local file ${FILENAME}..." - \rm -f $FILENAME - else - ## Extract the path part from the uri, and sanitize it. - ## "//" are not accepted by dirac commands. - local LFN=`echo "${URI}" | sed -r -e 's%^\w+://[^/]*(/[^?]+)(\?.*)?$%\1%' -e 's#//#/#g'` - - if [ "${TEST}" = true ] - then - LFN=${LFN}-uploadTest - fi - - info "Deleting file ${LFN}..." - dirac-dms-remove-files ${LFN} - fi - - stopLog file_delete -} diff --git a/src/main/resources/vm/script/datamanagement/downloadFunctions.vm b/src/main/resources/vm/script/datamanagement/downloadFunctions.vm deleted file mode 100644 index 98d4aaeb..00000000 --- a/src/main/resources/vm/script/datamanagement/downloadFunctions.vm +++ /dev/null @@ -1,281 +0,0 @@ -## downloadFunctions.vm - -## Variables used: -## $timeout, $minAvgDownloadThroughput, $bdiiTimeout, $srmTimeout -## Variables received, but no more used: -## $failOverEnabled, $failOverHost, $failOverPort, $failOverHome - -function downloadLFN { - - local LFN=$1 - - # Sanitize LFN: - # - "lfn:" at the beginning is optional for dirac-dms-* commands, - # but does not work as expected with comdirac commands like - # dmkdir. - # - "//" are not accepted, neither by dirac-dms-*, nor by dmkdir. - LFN=$(echo ${LFN} | sed -r -e 's/^lfn://' -e 's#//#/#g') - - info "getting file size and computing sendReceiveTimeout" - local size=$(dirac-dms-lfn-metadata ${LFN} | grep Size | sed -r 's/.* ([0-9]+)L,/\1/') - ## The $ sign must not be interpreted by velocity in the following - ## shell line. - #set ( $D = '$' ) - local sendReceiveTimeout=`echo ${D}[${D}{size:-0}/${minAvgDownloadThroughput}/1024]` - if [ "$sendReceiveTimeout" = "" ] || [ $sendReceiveTimeout -le 900 ] - then - info "sendReceiveTimeout empty or too small, setting it to 900s" - sendReceiveTimeout=900 - else - info "sendReceiveTimeout is $sendReceiveTimeout" - fi - - local LOCAL=${PWD}/`basename ${LFN}` - info "Removing file ${LOCAL} in case it is already here" - \rm -f ${LOCAL} - - local totalTimeout=$((${timeout} + ${srmTimeout} + ${sendReceiveTimeout})) - - local LINE="dirac-dms-get-file -d -o /Resources/StorageElements/GFAL_TIMEOUT=${totalTimeout} ${LFN}" - info ${LINE} - (${LINE}) &> get-file.log - - if [ $? = 0 ] - then - info "dirac-dms-get-file worked fine" - local source=$(grep "generating url" get-file.log | tail -1 | sed -r 's/^.* (.*)\.$/\1/') - info "DownloadCommand=dirac-dms-get-file Source=${source} Destination=$(hostname) Size=${size}" - RET_VAL=0 - else - error "dirac-dms-get-file failed" - error "`cat get-file.log`" - RET_VAL=1 - fi - - \rm get-file.log - return ${RET_VAL} -} -export -f downloadLFN - -# -# URI are of the form of the following example. A single "/", instead -# of 3, after "girder:" is also allowed. -# girder:///control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69 -# -# -# The code is quite the same as the uploadGirderFile function. Any -# changes should be done the same in both functions. -# -function downloadGirderFile { - local URI=$1 - - # The regexpes are written so that case is ignored and the - # arguments can be in any order. - local fileName=`echo $URI | sed -r 's#^girder:/(//)?([^/].*)\?.*$#\2#i'` - local apiUrl=`echo $URI | sed -r 's/^.*[?&]apiurl=([^&]*)(&.*)?$/\1/i'` - local fileId=`echo $URI | sed -r 's/^.*[?&]fileid=([^&]*)(&.*)?$/\1/i'` - local token=`echo $URI | sed -r 's/^.*[?&]token=([^&]*)(&.*)?$/\1/i'` - - if [ ! `which girder-client` ] - then - pip install --user girder-client - if [ $? != 0 ] - then - error "girder-client not in PATH, and an error occured while trying to install it." - error "Exiting with return value 1" - exit 1 - fi - fi - - COMMLINE="girder-client --api-url ${apiUrl} --token ${token} download --parent-type file ${fileId} ./${fileName}" - echo "downloadGirderFile, command line is ${COMMLINE}" - ${COMMLINE} -} -export -f downloadGirderFile - - -#This function identifies the gfal path and extracts the basename of the directory to be mounted, and creates a directory with the exact name on $PWD of the node. -#This directory gets mounted with the corresponding directory on the SE. - -#check_mount checks for all the gfal mounts in the current folder -check_mount='$(test -z $(for file in *; do findmnt -t fuse.gfalFS -lo Target -n -T $(realpath ${file}); done) && echo 1 || echo 0)' -isGfalmountExec=1 -function mountGfal { - local URI=$1 - - # The regexpes are written so that case is ignored and the - # arguments can be in any order. - local fileName=`echo $URI | sed -r 's#^srm:/(//)?([^/].*)\?.*$#\2#i'` - local gfal_basename=$(basename ${fileName}) - local job_id=${gfal_basename}_$(basename $PWD) - - CREATE_DIR_COMMAND="mkdir -p $gfal_basename" - SYM_LINK_COMMAND="ln -s $PWD/$gfal_basename /tmp/$job_id" - GFAL_COMMAND="gfalFS -s /tmp/$job_id ${fileName}" - - ${CREATE_DIR_COMMAND} - ${SYM_LINK_COMMAND} - ${GFAL_COMMAND} - #let nfs-kernel-server export the directory and write logs - sleep 30 - eval echo $check_mount -} - -export -f mountGfal - - -#This function un-mounts all the gfal mounted directories by searching them with 'findmnt' and filtering them with FSTYPE 'fuse.gfalFS' -#This function gets called in the cleanup function, either after the execution of the job, failure of the job or interruptions of the job - -function unmountGfal { - START=$SECONDS - while [ $(eval echo $check_mount) = 0 ] - do - for file in $PWD/* ;do findmnt -t fuse.gfalFS -lo Target -n -T $(realpath ${file}) && gfalFS_umount $(realpath ${file}); done - sleep 2 - if [[ $SECONDS-$START -gt 600 ]] #while loops breaks in automatically after 10 mins - then - echo "WARNING -gfal directory couldn't be unmounted:timeout" - break - fi - done - eval echo $check_mount -} - -export -f unmountGfal - - -# -# URI are of the form of the following example. A single "/", instead -# of 3, after "shanoir:" is also allowed. -# shanoir:/download.dcm?apiurl=https://shanoir-ng-nginx/shanoir-ng/datasets/carmin-data/path&format=dcm&resourceId=1 -# -# This method depends on refresh token process to refresh the token when it needs -# -function downloadShanoirFile { - local URI=$1 - - wait_for_token - - local token=`cat $SHANOIR_TOKEN_LOCATION` - - echo "token inside download : ${token}" - - local fileName=`echo $URI | sed -r 's#^shanoir:/(//)?([^/].*)\?.*$#\2#i'` - local apiUrl=`echo $URI | sed -r 's/^.*[?&]apiurl=([^&]*)(&.*)?$/\1/i'` - local format=`echo $URI | sed -r 's/^.*[?&]format=([^&]*)(&.*)?$/\1/i'` - local resourceId=`echo $URI | sed -r 's/^.*[?&]resourceId=([^&]*)(&.*)?$/\1/i'` - local converterId=`echo $URI | sed -r 's/^.*[?&]converterId=([^&]*)(&.*)?$/\1/i'` - - COMMAND(){ - curl --write-out '%{http_code}' -o ${fileName} --request GET "${apiUrl}/${resourceId}?format=${format}&converterId=${converterId}" --header "Authorization: Bearer ${token}" - } - - local attempts=0 - - while [[ "${attempts}" -ne 3 ]]; do - status_code=$(COMMAND) - info "downloadShanoirFIle, status code is : ${status_code}" - - if [[ "$status_code" -ne 200 ]]; then - error "error while downloading the file with status : ${status_code}" - attempts=$((attempts + 1)) - info "${attempts} done. Waiting 3 seconds and maybe do another attempt" - sleep 3 - else - break - fi - done - - if [[ "${attempts}" -ge 3 ]]; then - error "3 failures at downloading, stop trying and stop the job" - stopRefreshingToken - exit 1 - fi - - # if [[ $format = "compressed-nifti" ]]; then - if [[ $format = "nii" ]]; then - echo "its a nifti, shanoir has zipped it" - TMP_UNZIP_DIR="tmp_unzip_dir" - mkdir $TMP_UNZIP_DIR - mv $fileName $TMP_UNZIP_DIR/tmp.zip - unzip -d $TMP_UNZIP_DIR $TMP_UNZIP_DIR/tmp.zip - # there should be a unique .nii ou .nii.gz file somewhere - searchResult=$(find $TMP_UNZIP_DIR -name '*.nii.gz' -o -name '*.nii') - # doing this trick instead of using "wc -l" because it fails when there is no result - if [[ $(echo -n "$searchResult" | grep -c '^') -ne 1 ]]; then - error "too many or none nifti file (.nii or .nii.gz) in shanoir zip, supporting only 1" - stopRefreshingToken - exit 1 - fi - mv "$searchResult" "$fileName" - rm -rf $TMP_UNZIP_DIR - fi -} - -function downloadURI { - - local URI=$1 - local URI_LOWER=`echo $1 | awk '{print tolower($0)}'` - - if [[ ${URI_LOWER} == lfn* ]] || [[ $URI_LOWER == /* ]] - then - ## Extract the path part from the uri, and remove // if - ## present in path. - LFN=`echo "${URI}" | sed -r -e 's%^\w+://[^/]*(/[^?]+)(\?.*)?$%\1%' -e 's#//#/#g'` - - checkCacheDownloadAndCacheLFN $LFN - validateDownload "Cannot download LFN file" - fi - - if [[ ${URI_LOWER} == file:/* ]] - then - local FILENAME=`echo $URI | sed 's%file://*%/%'` - cp $FILENAME . - validateDownload "Cannot copy input file: $FILENAME" - fi - - if [[ ${URI_LOWER} == http://* ]] - then - curl --insecure -O ${URI} - validateDownload "Cannot download HTTP file" - fi - - if [[ ${URI_LOWER} == girder:/* ]] - then - downloadGirderFile ${URI} - validateDownload "Cannot download Girder file" - fi - - if [[ ${URI_LOWER} == shanoir:/* ]] - then - if [[ "$REFRESHING_JOB_STARTED" == false ]]; then - #set( $D = '$' ) - refresh_token ${URI} & - REFRESH_PID=${D}! - REFRESHING_JOB_STARTED=true - fi - downloadShanoirFile ${URI} - validateDownload "Cannot download shanoir file" - fi - - if [[ ${URI_LOWER} == srm:/* ]] - then - if [[ $(mountGfal ${URI}) -eq 0 ]] - then - isGfalmountExec=0 - else - echo "Cannot download gfal file" - fi - fi -} - -function validateDownload() { - - if [ $? != 0 ] - then - error "$1" - error "Exiting with return value 1" - exit 1 - fi -} diff --git a/src/main/resources/vm/script/datamanagement/refresh.vm b/src/main/resources/vm/script/datamanagement/refresh.vm deleted file mode 100644 index 23113ef2..00000000 --- a/src/main/resources/vm/script/datamanagement/refresh.vm +++ /dev/null @@ -1,103 +0,0 @@ -## refresh.vm - - -SHANOIR_TOKEN_LOCATION="${PWD}/cache/SHANOIR_TOKEN.txt" -SHANOIR_REFRESH_TOKEN_LOCATION="${PWD}/cache/SHANOIR_REFRESH_TOKEN.txt" -REFRESHING_JOB_STARTED=false - -REFRESH_PID="" - -# -# this is a background process to refresh shanoir token -# URI are of the form of the following example. A single "/", instead -# of 3, after "shanoir:" is also allowed. -# shanoir:/path/to/file/filename?&refreshToken=eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lk....&keycloak_client_id=....&keycloak_client_secret=... -# the mandatory arguments are : keycloak_client_id, keycloak_client_secret. -# -function refresh_token { - touch $SHANOIR_TOKEN_LOCATION - touch $SHANOIR_REFRESH_TOKEN_LOCATION - - local subshell_refresh_token=`cat $SHANOIR_REFRESH_TOKEN_LOCATION` - - echo "refresh token process started !" - - local URI=$1 - local keycloak_client_id=`echo $URI | sed -r 's/^.*[?&]keycloak_client_id=([^&]*)(&.*)?$/\1/i'` - local refresh_token_url=`echo $URI | sed -r 's/^.*[?&]refresh_token_url=([^&]*)(&.*)?$/\1/i'` - - if [[ !"$subshell_refresh_token" ]]; then - # initializing the refresh token - subshell_refresh_token=`echo $URI | sed -r 's/^.*[?&]refreshToken=([^&]*)(&.*)?$/\1/i'` - echo $subshell_refresh_token > $SHANOIR_REFRESH_TOKEN_LOCATION - fi - - while :; do - - # get the new refresh token - subshell_refresh_token=`cat $SHANOIR_REFRESH_TOKEN_LOCATION` - - # the response format is "{"status":"status"}" - # this response format is made to handle error while getting the refreshed token in the same time - COMMAND(){ - curl -w "{\"status\":\"%{http_code}\"}" -sb -o --request POST "${refresh_token_url}" --header "Content-Type: application/x-www-form-urlencoded" --data-urlencode "client_id=${keycloak_client_id}" --data-urlencode "grant_type=refresh_token" --data-urlencode "refresh_token=${subshell_refresh_token}" - } - - refresh_response=$(COMMAND) - status_code=`echo $refresh_response | grep -o '"status":"[^"]*' | grep -o '[^"]*$'` - - if [[ "$status_code" -ne 200 ]]; then - error_message=`echo $refresh_response | grep -o '"error_description":"[^"]*' | grep -o '[^"]*$'` - error "error while refreshing the token with status : ${status_code} and message error : ${error_message}" - exit 1 - fi - - # setting the new tokens - echo $refresh_response | grep -o '"access_token":"[^"]*' | grep -o '[^"]*$' > $SHANOIR_TOKEN_LOCATION - echo $refresh_response | grep -o '"refresh_token":"[^"]*' | grep -o '[^"]*$' > $SHANOIR_REFRESH_TOKEN_LOCATION - - sleep 240 - done - -} -# -# cleanup method : stop the refreshing process -# -function stopRefreshingToken { - if [ "${REFRESH_PID}" != "" ] - then - info "Killing background refresh token process with id : ${REFRESH_PID}" - kill -9 ${REFRESH_PID} - REFRESH_PID="" - echo "refresh token process ended !" - fi -} - -# -# the refresh token may take some time, this method is for that purpose -# and it exit the program if it's timedout -# -function wait_for_token { - local token="" - local attempts=0 - - while [[ "${attempts}" -ne 3 ]]; do - token=`cat $SHANOIR_TOKEN_LOCATION` - if [[ "${token}" == "" ]]; then - echo "token is not refreshed yet, waitting for 3 seconds..." - echo "attempts : ${attempts}" - attempts=$((attempts + 1)) - sleep 3 - else - echo "token is refreshed !" - break - fi - done - - ## check the token after the time out - if [[ "${token}" == "" ]]; then - error "token refreshing is taking too long, abording the process" - stopRefreshingToken - exit 1 - fi -} diff --git a/src/main/resources/vm/script/datamanagement/uploadFunctions.vm b/src/main/resources/vm/script/datamanagement/uploadFunctions.vm deleted file mode 100644 index 6d1d6f87..00000000 --- a/src/main/resources/vm/script/datamanagement/uploadFunctions.vm +++ /dev/null @@ -1,279 +0,0 @@ -## uploadFunctions.vm - -## Variables used: -## $timeout, $minAvgDownloadThroughput, $bdiiTimeout, $srmTimeout -## Variables received, but no more used: -## $failOverEnabled - -function nSEs { - - i=0 - for n in ${SELIST} - do - i=`expr $i + 1` - done - return $i -} - -function getAndRemoveSE { - - local index=$1 - local i=0 - local NSE="" - RESULT="" - for n in ${SELIST} - do - if [ "$i" = "${index}" ] - then - RESULT=$n - info "result: $RESULT" - else - NSE="${NSE} $n" - fi - i=`expr $i + 1` - done - SELIST=${NSE} - return 0 -} - -function chooseRandomSE { - - nSEs - local n=$? - if [ "$n" = "0" ] - then - info "SE list is empty" - RESULT="" - else - local r=${RANDOM} - local id=`expr $r % $n` - getAndRemoveSE ${id} - fi -} - -function uploadLfnFile { - - local LFN=$1 - local FILE=$2 - local nrep=$3 - local SELIST=${SE} - - # Sanitize LFN: - # - "lfn:" at the beginning is optional for dirac-dms-* commands, - # but does not work as expected with comdirac commands like - # dmkdir. - # - "//" are not accepted, neither by dirac-dms-*, nor by dmkdir. - LFN=$(echo ${LFN} | sed -r -e 's/^lfn://' -e 's#//#/#g') - - info "getting file size and computing sendReceiveTimeout" - local size=`ls -l ${FILE} | awk -F' ' '{print $5}'` - ## The $ sign must not be interpreted by velocity in the following - ## shell line. - #set ( $D = '$' ) - local sendReceiveTimeout=`echo ${D}[${D}{size:-0}/${minAvgDownloadThroughput}/1024]` - if [ "$sendReceiveTimeout" = "" ] || [ $sendReceiveTimeout -le 900 ] - then - info "sendReceiveTimeout empty or too small, setting it to 900s" - sendReceiveTimeout=900 - else - info "sendReceiveTimeout is $sendReceiveTimeout" - fi - - local totalTimeout=$((${timeout} + ${srmTimeout} + ${sendReceiveTimeout})) - - local OPTS="-o /Resources/StorageElements/GFAL_TIMEOUT=${totalTimeout}" - chooseRandomSE - local DEST=${RESULT} - local done=0 - while [ $nrep -gt $done ] && [ "${DEST}" != "" ] - do - if [ "${done}" = "0" ] - then - local command="dirac-dms-add-file" - local source=$(hostname) - dirac-dms-remove-files ${OPTS} ${LFN} &>/dev/null - (dirac-dms-add-file ${OPTS} ${LFN} ${FILE} ${DEST}) &> dirac.log - local error_code=$? - else - local command="dirac-dms-replicate-lfn" - (dirac-dms-replicate-lfn -d ${OPTS} ${LFN} ${DEST}) &> dirac.log - local error_code=$? - - # Extract the name of the source SE from the logs. - local source=$(grep "operation 'getFileSize'" dirac.log | tail -1 | sed -r 's/^.* StorageElement (.*) is .*$/\1/') - fi - if [ ${error_code} = 0 ] - then - info "Copy/Replication of ${LFN} to SE ${DEST} worked fine." - done=`expr ${done} + 1` - info "UploadCommand=${command} Source=${source} Destination=${DEST} Size=${size}" - else - error "`cat dirac.log`" - warning "Copy/Replication of ${LFN} to SE ${DEST} failed" - fi - \rm dirac.log - chooseRandomSE - DEST=${RESULT} - done - if [ "${done}" = "0" ] - then - error "Cannot copy file ${FILE} to lfn ${LFN}" - error "Exiting with return value 2" - exit 2 - else - addToCache ${LFN} ${FILE} - fi -} - -# -# This method is used to upload results of an execution to an upload url. -# URI are of the form of the following example. A single "/", instead -# of 3, after "shanoir:" is also allowed. -# shanoir:/path/to/file/filename?upload_url=https://upload/url/&type=File&md5=None -# -# This method depends on refresh token process to refresh the token when it needs -# -function uploadShanoirFile { - local URI=$1 - - wait_for_token - - local token=`cat $SHANOIR_TOKEN_LOCATION` - - local upload_url=`echo $URI | sed -r 's/^.*[?&]upload_url=([^&]*)(&.*)?$/\1/i'` - local fileName=`echo $URI | sed -r 's#^shanoir:/(//)?(.*/(.+))\?.*$#\3#i'` - local filePath=`echo $URI | sed -r 's#^shanoir:/(//)?([^/].*)\?.*$#\2#i'` - - local type=`echo $URI | sed -r 's/^.*[?&]type=([^&]*)(&.*)?$/\1/i'` - local md5=`echo $URI | sed -r 's/^.*[?&]md5=([^&]*)(&.*)?$/\1/i'` - - - COMMAND(){ - (echo -n '{"base64Content": "'; base64 ${fileName}; echo '", "type":"'; echo ${type}; echo '", "md5":"'; echo ${md5} ; echo '"}') | curl --write-out '%{http_code}' --request PUT "${upload_url}/${filePath}" --header "Authorization: Bearer ${token}" --header "Content-Type: application/carmin+json" --header 'Accept: application/json, text/plain, */*' -d @- - } - - - status_code=$(COMMAND) - echo "uploadShanoirFIle, status code is : ${status_code}" - - if [[ "$status_code" -ne 201 ]]; then - error "error while uploading the file with status : ${status_code}" - stopRefreshingToken - exit 1 - fi -} - - -# -# URI are of the form of the following example. A single "/", instead -# of 3, after "girder:" is also allowed. -# girder:///control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69 -# -# The code is quite the same as the downloadGirderFile function. Any -# changes should be done the same in both functions. -# -function uploadGirderFile { - local URI=$1 - - # The regexpes are written so that case is ignored and the - # arguments can be in any order. - local fileName=`echo $URI | sed -r 's#^girder:/(//)?(.*/)?([^/].*)\?.*$#\3#i'` - local apiUrl=`echo $URI | sed -r 's/^.*[?&]apiurl=([^&]*)(&.*)?$/\1/i'` - local fileId=`echo $URI | sed -r 's/^.*[?&]fileid=([^&]*)(&.*)?$/\1/i'` - local token=`echo $URI | sed -r 's/^.*[?&]token=([^&]*)(&.*)?$/\1/i'` - - if [ ! `which girder-client` ] - then - pip install --user girder-client - if [ $? != 0 ] - then - error "girder-client not in PATH, and an error occured while trying to install it." - error "Exiting with return value 1" - exit 1 - fi - fi - - COMMLINE="girder-client --api-url ${apiUrl} --token ${token} upload --parent-type folder ${fileId} ./${fileName}" - echo "uploadGirderFile, command line is ${COMMLINE}" - ${COMMLINE} - if [ $? != 0 ] - then - error "Error while uploading girder file" - error "Exiting with return value 1" - exit 1 - fi -} - -function upload { - - local URI=$1 - local ID=$2 - local NREP=$3 - local TEST=$4 - - startLog file_upload id="$ID" uri="$URI" - - ## The pattern must NOT be put between quotation marks. - if [[ ${URI} == shanoir:/* ]] - then - if [ "${TEST}" != "true" ] - then - if [[ "$REFRESHING_JOB_STARTED" == false ]]; then - #set( $D = '$' ) - refresh_token ${URI} & - REFRESH_PID=${D}! - REFRESHING_JOB_STARTED=true - fi - uploadShanoirFile ${URI} - fi - elif [[ ${URI} == girder:/* ]] - then - if [ "${TEST}" != "true" ] - then - uploadGirderFile ${URI} - fi - elif [[ ${URI} == file:/* ]] - then - local FILENAME=`echo $URI | sed 's%file://*%/%'` - local NAME=`basename ${FILENAME}` - - if [ -e $FILENAME ] - then - error "Result file already exists: $FILENAME" - error "Exiting with return value 1" - exit 1 - fi - - if [ "${TEST}" = "true" ] - then - echo "test result" > ${NAME} - fi - - \mv $NAME $FILENAME - if [ $? != 0 ] - then - error "Error while moving result local file." - error "Exiting with return value 1" - exit 1 - fi - else - ## Extract the path part from the uri. - local LFN=`echo "${URI}" | sed -r 's%^\w+://[^/]*(/[^?]+)(\?.*)?$%\1%'` -#set( $fileName = '${LFN##*/}' ) - local NAME=${fileName} - - if [ "${TEST}" = "true" ] - then - LFN=${LFN}-uploadTest - echo "test result" > ${NAME} - fi - - uploadLfnFile ${LFN} ${PWD}/${NAME} ${NREP} - - if [ "${TEST}" = "true" ] - then - \rm -f ${NAME} - fi - fi - stopLog file_upload -} diff --git a/src/main/resources/vm/script/execution/backgroundScript.vm b/src/main/resources/vm/script/execution/backgroundScript.vm deleted file mode 100644 index bebdfa6c..00000000 --- a/src/main/resources/vm/script/execution/backgroundScript.vm +++ /dev/null @@ -1,16 +0,0 @@ -## backgroundScript.vm - -## Variables -## $minorStatusEnabled, $serviceCall, $backgroundScript - -startLog background - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 2 -#end - -checkCacheDownloadAndCacheLFN $backgroundScript -bash `basename $backgroundScript` 1>background.out 2>background.err & -BACKPID=$! - -stopLog background \ No newline at end of file diff --git a/src/main/resources/vm/script/execution/execution.vm b/src/main/resources/vm/script/execution/execution.vm deleted file mode 100644 index 72cf41e1..00000000 --- a/src/main/resources/vm/script/execution/execution.vm +++ /dev/null @@ -1,69 +0,0 @@ -## execution.vm - -## Variables -## $minorStatusEnabled, $serviceCall, $executableName, $params - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 4 -#end - -#set ( $tarFile = "${executableName}.tar.gz" ) -tar -zxf $tarFile - -chmod 755 * - -## the 1s delay is needed to ensure that the time between this file creation and the command line outputs -## files creation is sufficient, and the subsequent "find -newer" call succeeds - -echo "BEFORE_EXECUTION_REFERENCE" > BEFORE_EXECUTION_REFERENCE_FILE -sleep 1 - -#set( $parameters = "" ) -#foreach( $param in $params ) - #set( $parameters = "$parameters $param" ) -#end - -export LD_LIBRARY_PATH=${PWD}:${LD_LIBRARY_PATH} - -## Set HOME if not defined. -## Also set APPTAINER_HOME, so that HOME is set inside singularity containers. -if [ -z "${HOME}" ]; then - export HOME="${PWD}" - export APPTAINER_HOME="${PWD}:${PWD}" -fi - -## The command_line variable is an array to allow spaces in string inputs -COMMAND_LINE=(./$executableName $parameters) - -info "Executing $COMMAND_LINE..." - -startLog application_execution - -#[[ -"${COMMAND_LINE[@]}" -]]# - -if [ $? -ne 0 ] -then - error "Exiting with return value 6" - BEFOREUPLOAD=`date +%s` - info "Execution time: `expr ${BEFOREUPLOAD} - ${AFTERDOWNLOAD}` seconds" - stopLog application_execution - cleanup - exit 6 -fi -BEFOREUPLOAD=`date +%s` -stopLog application_execution - -info "Execution time was `expr ${BEFOREUPLOAD} - ${AFTERDOWNLOAD}`s" - -PROVENANCE_DEST="$BASEDIR/$DIRNAME.sh.provenance.json" -info "copying provenance file to $PROVENANCE_DEST" -if [ -f provenance.json ]; then - cp provenance.json "$PROVENANCE_DEST" -else - warning "provenance.json not found" -fi - -__MOTEUR_ARGS="$parameters" -__MOTEUR_EXE="$executableName" \ No newline at end of file diff --git a/src/main/resources/vm/script/execution/footer.vm b/src/main/resources/vm/script/execution/footer.vm deleted file mode 100644 index 75c06cb6..00000000 --- a/src/main/resources/vm/script/execution/footer.vm +++ /dev/null @@ -1,26 +0,0 @@ -## footer.vm - -## Variables -## $minorStatusEnabled, $serviceCall - -startLog footer - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 6 -#end - -cleanup -STOP=`date +%s` -info "Stop date is ${STOP}" -TOTAL=`expr $STOP - $START` -info "Total running time: $TOTAL seconds" -UPLOAD=`expr ${STOP} - ${BEFOREUPLOAD}` -DOWNLOAD=`expr ${AFTERDOWNLOAD} - ${START}` -info "Input download time: ${DOWNLOAD} seconds" -info "Execution time: `expr ${BEFOREUPLOAD} - ${AFTERDOWNLOAD}` seconds" -info "Results upload time: ${UPLOAD} seconds" -info "Exiting with return value 0" -info "(HACK for ARC: writing it in ${DIAG})" -info "exitcode=0" >> ${DIAG} -exit 0 -stopLog footer \ No newline at end of file diff --git a/src/main/resources/vm/script/execution/header.vm b/src/main/resources/vm/script/execution/header.vm deleted file mode 100644 index 4539957d..00000000 --- a/src/main/resources/vm/script/execution/header.vm +++ /dev/null @@ -1,65 +0,0 @@ -## header.vm - -## Variables -## $minorStatusEnabled, $serviceCall, $defaultEnvironment, $voDefaultSE, -## $voUseCloseSE, $simulationID, $boshCVMFSPath, $containersCVMFSPath, $udockerTag - -startLog header - -START=`date +%s` -info "START date is ${START}" - -## Execution environment setup -export GASW_JOB_ENV=NORMAL -export GASW_EXEC_ENV=EGEE - -## Builds the custom environment -export BASEDIR=${PWD} -ENV=$defaultEnvironment -export $ENV -__MOTEUR_ENV=$defaultEnvironment -export SE=$voDefaultSE -USE_CLOSE_SE=$voUseCloseSE -export BOSH_CVMFS_PATH=$boshCVMFSPath -export CONTAINERS_CVMFS_PATH=$containersCVMFSPath -export UDOCKER_TAG=$udockerTag -export BOUTIQUES_PROV_DIR=$boutiquesProvenanceDir - -export MOTEUR_WORKFLOWID="$simulationID" - -## if the execution environment is a cluster, the vlet binaries should be added to the path -if [[ "$GASW_EXEC_ENV" == "PBS" ]] -then - export PATH=${VLET_INSTALL}/bin:$PATH -fi - -DIAG=/home/grid/session/`basename ${PWD}`.diag; - -## Creates execution directory -DIRNAME=`basename $0 .sh` -mkdir ${DIRNAME} -if [ $? = 0 ] -then - echo "cd ${DIRNAME}" - cd ${DIRNAME} -else - echo "Unable to create directory ${DIRNAME}" - echo "Exiting with return value 7" - exit 7 -fi - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 1 -#end - -BACKPID="" - -#DIRAC may wrongly position this variable -test -d ${X509_CERT_DIR} -if [ $? != 0 ] -then - info "Unsetting invalid X509_CERT_DIR (${X509_CERT_DIR})" - unset X509_CERT_DIR -fi - -stopLog header diff --git a/src/main/resources/vm/script/execution/hostConfiguration.vm b/src/main/resources/vm/script/execution/hostConfiguration.vm deleted file mode 100644 index 3742e237..00000000 --- a/src/main/resources/vm/script/execution/hostConfiguration.vm +++ /dev/null @@ -1,36 +0,0 @@ -## hostConfiguration.vm - -## Variables -## $cacheDir - -startLog host_config - -echo "SE Linux mode is:" -/usr/sbin/getenforce -echo gLite Job Id is ${GLITE_WMS_JOBID} -echo "===== uname ===== " - uname -a - domainname -a -echo "===== network config ===== " - /sbin/ifconfig eth0 - dmesg_line=$(dmesg | grep 'Link is Up' | uniq) - netspeed=$(echo $dmesg_line | grep -o '[0-9]*[[:space:]][a-zA-Z]bps'| awk '{gsub(/ /,"",$0);print}') - echo "NetSpeed = $netspeed ($dmesg_line)" -echo "===== CPU info ===== " - cat /proc/cpuinfo -echo "===== Memory info ===== " - cat /proc/meminfo -echo "===== lcg-cp location ===== " - which lcg-cp; -echo "===== ls -a . ===== " - ls -a -echo "===== ls -a .. ===== " - ls -a .. -echo "===== env =====" - env -echo "===== rpm -qa ====" - rpm -qa - -mkdir -p $cacheDir - -stopLog host_config diff --git a/src/main/resources/vm/script/execution/inputs.vm b/src/main/resources/vm/script/execution/inputs.vm deleted file mode 100644 index f1eacb90..00000000 --- a/src/main/resources/vm/script/execution/inputs.vm +++ /dev/null @@ -1,25 +0,0 @@ -## inputs.vm - -## Variables -## $minorStatusEnabled, $serviceCall, $downloads - -startLog inputs_download - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 3 -#end - -touch ../DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK - -#foreach( $download in $downloads ) - -startLog file_download uri="${download}" -downloadURI "$download" -__MOTEUR_IN="${__MOTEUR_IN};$download" -stopLog file_download -#end - -chmod 755 * -AFTERDOWNLOAD=`date +%s`; - -stopLog inputs_download \ No newline at end of file diff --git a/src/main/resources/vm/script/execution/result.vm b/src/main/resources/vm/script/execution/result.vm deleted file mode 100644 index 6e8e47f3..00000000 --- a/src/main/resources/vm/script/execution/result.vm +++ /dev/null @@ -1,29 +0,0 @@ -## result.vm - -## Variables -## $minorStatusEnabled, $serviceCall, $uploads - -startLog results_upload - -#if( $minorStatusEnabled && $serviceCall ) -$serviceCall ${MOTEUR_WORKFLOWID} ${JOBID} 5 -#end - -#set( $uploadsList = "" ) -#set( $count = 0 ) -#foreach( $upload in $uploads ) - #set( $URI = "$upload.URI" ) - #if( $count > 0 ) - #set( $uploadsList = "$uploadsList;$URI" ) - #else - #set( $uploadsList = $URI ) - #end - -# Redirecting tr errors to /dev/null to avoid sometimes a normal -# broken pipe error. -upload "$URI" "$upload.getId()" $upload.NumberOfReplicas false -#end - -__MOTEUR_OUT="$uploadsList" - -stopLog results_upload \ No newline at end of file diff --git a/src/main/resources/vm/script/execution/uploadTest.vm b/src/main/resources/vm/script/execution/uploadTest.vm deleted file mode 100644 index cf0b3e11..00000000 --- a/src/main/resources/vm/script/execution/uploadTest.vm +++ /dev/null @@ -1,20 +0,0 @@ -## uploadTest.vm - -## Variables -## $cacheDir, $uri, $nrep - -startLog upload_test - -## creates void result -mkdir -p $cacheDir -test -f $cacheDir/uploadChecked -if [ $? != 0 ] -then - upload "$uri" "" $nrep true - delete "$uri" true - touch $cacheDir/uploadChecked -else - info "Skipping upload test (it has already been done by a previous job)" -fi - -stopLog upload_test diff --git a/src/main/resources/vm/script/execution/variables.vm b/src/main/resources/vm/script/execution/variables.vm deleted file mode 100644 index 524215a6..00000000 --- a/src/main/resources/vm/script/execution/variables.vm +++ /dev/null @@ -1,12 +0,0 @@ -## variables.vm - -## Variables -## $variables - -startLog application_environment - -#foreach( $variable in $variables.entrySet() ) - export $variable.key="$variable.value" -#end - -stopLog application_environment \ No newline at end of file diff --git a/src/test/java/fr/insalyon/creatis/gasw/DatabaseConfigurationTest.java b/src/test/java/fr/insalyon/creatis/gasw/DatabaseConfigurationTest.java new file mode 100644 index 00000000..a1651375 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/DatabaseConfigurationTest.java @@ -0,0 +1,124 @@ +package fr.insalyon.creatis.gasw; + +import com.zaxxer.hikari.HikariDataSource; +import fr.insalyon.creatis.gasw.dao.hibernate.*; +import fr.insalyon.creatis.gasw.plugin.DatabasePlugin; +import jakarta.persistence.EntityManagerFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.orm.jpa.JpaTransactionManager; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.Transactional; + +import javax.sql.DataSource; +import java.lang.reflect.Method; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class DatabaseConfigurationTest { + + @Mock + private DatabasePlugin dbPlugin; + + private DatabaseConfiguration configuration; + + @BeforeEach + void setUp() { + configuration = new DatabaseConfiguration(dbPlugin); + } + + @Test + @DisplayName("dataSource() wires Hikari from the DatabasePlugin") + void dataSource_isConfiguredFromPlugin() { + when(dbPlugin.getDriverClass()).thenReturn("org.h2.Driver"); + when(dbPlugin.getUserName()).thenReturn("sa"); + when(dbPlugin.getPassword()).thenReturn("secret"); + when(dbPlugin.getConnectionUrl()).thenReturn("jdbc:h2:mem:test"); + + DataSource dataSource = configuration.dataSource(); + + assertInstanceOf(HikariDataSource.class, dataSource); + HikariDataSource hikari = (HikariDataSource) dataSource; + assertEquals("org.h2.Driver", hikari.getDriverClassName()); + assertEquals("jdbc:h2:mem:test", hikari.getJdbcUrl()); + } + + @Test + @DisplayName("transactionManager() wraps the exact EntityManagerFactory bean it was given") + void transactionManager_wrapsGivenEntityManagerFactory() { + EntityManagerFactory emf = mock(EntityManagerFactory.class); + + PlatformTransactionManager txManager = configuration.transactionManager(emf); + + assertInstanceOf(JpaTransactionManager.class, txManager); + assertSame(emf, ((JpaTransactionManager) txManager).getEntityManagerFactory()); + } + + @ParameterizedTest(name = "class={0}, method={1}, readOnly={2}") + @DisplayName("DAO methods have correct transactional settings") + @MethodSource("daoMethods") + void daoMethods_haveCorrectTransactionalSettings(Class daoClass, String methodName, boolean readOnlyExpected) throws Exception { + Transactional tx = findMethod(daoClass, methodName).getAnnotation(Transactional.class); + + assertNotNull(tx, methodName + " missing @Transactional"); + assertEquals(readOnlyExpected, tx.readOnly(), + methodName + " has wrong readOnly value"); + } + + private static Method findMethod(Class clazz, String name) throws NoSuchMethodException { + for (Method m : clazz.getDeclaredMethods()) { + if (m.getName().equals(name)) { + return m; + } + } + throw new NoSuchMethodException(clazz.getSimpleName() + "#" + name); + } + + static Stream daoMethods() { + return Stream.of( + + // Read-only + Arguments.of(JobData.class, "getJobByID", true), + Arguments.of(JobData.class, "getActiveJobs", true), + Arguments.of(JobData.class, "getJobs", true), + + Arguments.of(NodeData.class, "getNodeBySiteAndNodeName", true), + + Arguments.of(SEEntryPointData.class, "getByHostName", true), + + Arguments.of(DataToReplicateData.class, "get", true), + + Arguments.of(JobMinorStatusData.class, "getCheckpoints", true), + Arguments.of(JobMinorStatusData.class, "getExecutionMinorStatus", true), + Arguments.of(JobMinorStatusData.class, "getDateDiff", true), + + // Write + Arguments.of(JobData.class, "add", false), + Arguments.of(JobData.class, "update", false), + Arguments.of(JobData.class, "remove", false), + + Arguments.of(NodeData.class, "add", false), + + Arguments.of(DataData.class, "upsertData", false), + + Arguments.of(SEEntryPointData.class, "add", false), + + Arguments.of(DataToReplicateData.class, "add", false), + Arguments.of(DataToReplicateData.class, "update", false), + Arguments.of(DataToReplicateData.class, "remove", false), + + Arguments.of(JobMinorStatusData.class, "add", false) + ); + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/FailOverTest.java b/src/test/java/fr/insalyon/creatis/gasw/FailOverTest.java new file mode 100644 index 00000000..066d5c06 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/FailOverTest.java @@ -0,0 +1,129 @@ +package fr.insalyon.creatis.gasw; + +import fr.insalyon.creatis.gasw.bean.DataToReplicate; +import fr.insalyon.creatis.gasw.dao.DAOException; +import fr.insalyon.creatis.gasw.dao.DataToReplicateDAO; +import fr.insalyon.creatis.gasw.dao.SEEntryPointsDAO; + +import fr.insalyon.creatis.gasw.execution.FailOver; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.reflect.Method; +import java.net.URI; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class FailOverTest { + + @Mock + private GaswConfiguration config; + + @Mock + private DataToReplicateDAO dataToReplicateDAO; + + @Mock + private SEEntryPointsDAO seEntryPointDAO; + + private FailOver failOver; + + @BeforeEach + void setUp() { + failOver = new FailOver(config, dataToReplicateDAO, seEntryPointDAO); + } + + @Nested + @DisplayName("addData()") + class AddData { + + @Test + @DisplayName("skips file:// and http:// URIs (already local / directly reachable)") + void skipsLocallyReachableSchemes() { + failOver.addData(URI.create("file:///tmp/x")); + failOver.addData(URI.create("HTTP://host/x")); + verifyNoInteractions(dataToReplicateDAO); + } + + @Test + @DisplayName("registers grid-scheme URIs (srm/lfn/...) for replication") + void registersGridSchemeUris() throws DAOException { + URI uri = URI.create("srm://host/path"); + failOver.addData(uri); + ArgumentCaptor captor = ArgumentCaptor.forClass(DataToReplicate.class); + verify(dataToReplicateDAO).add(captor.capture()); + assertEquals(uri, captor.getValue().getUrl()); + } + + @Test + @DisplayName("List overload registers each entry individually") + void listOverload_registersEachEntry() throws DAOException { + failOver.addData(List.of(URI.create("srm://a"), URI.create("lfn://b"))); + verify(dataToReplicateDAO, times(2)).add(any()); + } + } + + @Nested + @DisplayName("FailOver run() logic") + class Run { + + @Test + @DisplayName("does nothing when failover is disabled") + void disabled_doesNothing() throws Exception { + enableFailover(false); + invokeRun(); + verifyNoInteractions(dataToReplicateDAO); + } + + @Test + @DisplayName("below max retries: increments the retry count and reschedules") + void belowMaxRetries_incrementsAndReschedules() throws Exception { + enableFailover(true); + when(config.getFailOverMaxRetry()).thenReturn(5); + DataToReplicate data = new DataToReplicate(URI.create("srm://host/path"), 0); + when(dataToReplicateDAO.get()).thenReturn(List.of(data)); + invokeRun(); + assertEquals(1, data.getRetries()); + verify(dataToReplicateDAO).update(data); + verify(dataToReplicateDAO, never()).remove(data); + } + + @Test + @DisplayName("at max retries: gives up and removes the entry instead of retrying forever") + void atMaxRetries_givesUpAndRemoves() throws Exception { + enableFailover(true); + when(config.getFailOverMaxRetry()).thenReturn(2); + DataToReplicate data = new DataToReplicate(URI.create("srm://host/path"), 1); + when(dataToReplicateDAO.get()).thenReturn(List.of(data)); + invokeRun(); + verify(dataToReplicateDAO).remove(data); + verify(dataToReplicateDAO, never()).update(any()); + } + + @Test + @DisplayName("a DAOException while listing pending replications does not kill the scheduled task") + void daoExceptionOnGet_isSwallowed() throws Exception { + enableFailover(true); + when(dataToReplicateDAO.get()).thenThrow(new DAOException("db down")); + assertDoesNotThrow(this::invokeRun); + } + + private void invokeRun() throws Exception { + Method method = FailOver.class.getDeclaredMethod("run"); + method.setAccessible(true); + method.invoke(failOver); + } + + private void enableFailover(boolean enabled) { + when(config.isFailOverEnabled()).thenReturn(enabled); + } + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswConfigurationTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswConfigurationTest.java new file mode 100644 index 00000000..5a6465a8 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswConfigurationTest.java @@ -0,0 +1,31 @@ +package fr.insalyon.creatis.gasw; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +public class GaswConfigurationTest { + + @Test + @DisplayName("all declared property keys resolve against gasw.properties") + void allPropertyKeys_resolveAgainstGaswProperties() { + try (AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext()) { + ctx.register(PlaceholderConfig.class, GaswConfiguration.class); + assertDoesNotThrow(ctx::refresh, + "a @Value key in GaswConfiguration has no matching entry in gasw.properties"); + } + } + + @Configuration + static class PlaceholderConfig { + @Bean + static PropertySourcesPlaceholderConfigurer placeholderConfigurer() { + return new PropertySourcesPlaceholderConfigurer(); + } + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswLauncherTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswLauncherTest.java new file mode 100644 index 00000000..aa5af789 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswLauncherTest.java @@ -0,0 +1,55 @@ +package fr.insalyon.creatis.gasw; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertySource; + +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +public class GaswLauncherTest { + + @Test + @DisplayName("does nothing when ~/.gasw does not exist") + void missingConfigDir_doesNothing(@TempDir Path tempHome) throws Exception { + System.setProperty("user.home", tempHome.toString()); + ConfigurableEnvironment env = mock(ConfigurableEnvironment.class); + invokeLoadExternalConfig(env); + verifyNoInteractions(env); + } + + @Test + @DisplayName("registers every .properties file in ~/.gasw ahead of the defaults, ignores everything else") + void loadsOnlyPropertiesFiles_withHighestPrecedence(@TempDir Path tempHome) throws Exception { + System.setProperty("user.home", tempHome.toString()); + Path gaswDir = Files.createDirectory(tempHome.resolve(".gasw")); + Files.writeString(gaswDir.resolve("override.properties"), "gasw.vo.name=biomed"); + Files.writeString(gaswDir.resolve("readme.txt"), "not a properties file"); + + MutablePropertySources sources = new MutablePropertySources(); + ConfigurableEnvironment env = mock(ConfigurableEnvironment.class); + when(env.getPropertySources()).thenReturn(sources); + invokeLoadExternalConfig(env); + + PropertySource ps = sources.stream() + .filter(p -> p.getName().contains("override.properties")) + .findFirst() + .orElseThrow(); + + assertEquals("biomed", ps.getProperty("gasw.vo.name")); + assertEquals(1, sources.stream().count()); + } + + private void invokeLoadExternalConfig(ConfigurableEnvironment env) throws Exception { + Method method = GaswLauncher.class.getDeclaredMethod("loadExternalConfig", ConfigurableEnvironment.class); + method.setAccessible(true); + method.invoke(null, env); + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswLoggerTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswLoggerTest.java index 6635b563..ba6e9418 100644 --- a/src/test/java/fr/insalyon/creatis/gasw/GaswLoggerTest.java +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswLoggerTest.java @@ -58,7 +58,7 @@ public void verifyProdConfigIsTheSameAsTheTestOne() throws IOException, URISynta // assert the size is only configured twice (stdout and stderr) Assertions.assertEquals(2, testConfig.stream().filter(line -> line.trim().startsWith("")).count()); testConfig = testConfig.stream().map(line -> { - if ( ! line.trim().startsWith("")) { + if (!line.trim().startsWith("")) { return line; } return line.replace("1KB", "100MB"); @@ -71,7 +71,7 @@ public void verifyProdConfigIsTheSameAsTheTestOne() throws IOException, URISynta public void testLogFileSizeLimit() throws IOException, InterruptedException { Logger logger = LoggerFactory.getLogger(Gasw.class); // 100 log lines should take around 10KB - for (int i = 0; i<100; i++) { + for (int i = 0; i < 100; i++) { // wait a little because logback has a timeout and do not verify the size if the logs are too close // in logback-test.xml, logcback si configured to test every millisecond Thread.sleep(2); @@ -86,7 +86,7 @@ public void testLogFileSizeLimit() throws IOException, InterruptedException { public void cleanLogFiles() throws IOException, JoranException { try (Stream stream = Files.list(Paths.get(""))) { stream - .filter(file -> ! Files.isDirectory(file)) + .filter(file -> !Files.isDirectory(file)) .filter(file -> file.getFileName().toString().startsWith("workflow")) .forEach(f -> { System.out.println("deleting : " + f.getFileName()); @@ -112,7 +112,7 @@ public void assertLogFiles() throws IOException { List logFiles; try (Stream stream = Files.list(Paths.get(""))) { logFiles = stream - .filter(file -> ! Files.isDirectory(file)) + .filter(file -> !Files.isDirectory(file)) .filter(file -> file.getFileName().toString().startsWith("workflow")) .collect(Collectors.toList()); diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswNotificationTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswNotificationTest.java new file mode 100644 index 00000000..88486063 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswNotificationTest.java @@ -0,0 +1,83 @@ +package fr.insalyon.creatis.gasw; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +@ExtendWith(MockitoExtension.class) +public class GaswNotificationTest { + + @Nested + @DisplayName("scheduling contract") + class Scheduling { + + @Test + @DisplayName("terminate() stops notifyIfReady() from doing further work") + void terminate_stopsScheduledNotification() throws Exception { + GaswNotification notification = new GaswNotification(); + Object client = mock(Object.class); + notification.setNotificationClient(client); + notification.addFinishedJob(output("job1")); + notification.terminate(); + invokeNotifyIfReady(notification); + + verifyNoInteractions(client); + } + + private void invokeNotifyIfReady(GaswNotification notification) throws Exception { + Method method = GaswNotification.class.getDeclaredMethod("notifyIfReady"); + method.setAccessible(true); + method.invoke(notification); + } + } + + @Nested + @DisplayName("job tracking") + class JobTracking { + + @Test + @DisplayName("getFinishedJobs() drains the queue exactly once, in order") + void getFinishedJobs_drainsQueueInOrder() { + GaswNotification notification = new GaswNotification(); + GaswOutput first = output("job1"); + GaswOutput second = output("job2"); + notification.addFinishedJob(first); + notification.addFinishedJob(second); + + assertEquals(List.of(first, second), notification.getFinishedJobs()); + assertTrue(notification.getFinishedJobs().isEmpty()); + } + + @Test + @DisplayName("addErrorJob() keeps only the latest failure per job ID") + void addErrorJob_keepsOnlyLatestFailurePerJob() { + GaswNotification notification = new GaswNotification(); + GaswOutput firstAttempt = outputWithStdErr("job1"); + GaswOutput retryAttempt = outputWithStdErr("job1"); + + notification.addErrorJob(firstAttempt); + notification.addErrorJob(retryAttempt); + + assertSame(retryAttempt, notification.getGaswOutputFromLastFailedJob("job1")); + } + } + + private GaswOutput output(String jobId) { + return new GaswOutput(jobId, GaswExitCode.EXECUTION_FAILED, "", Map.of(), null, null, null, null); + } + + private GaswOutput outputWithStdErr(String jobId) { + return new GaswOutput(jobId, GaswExitCode.EXECUTION_FAILED, "", Map.of(), null, null, null, + new java.io.File("stderr.txt")); + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswSpringConfigTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswSpringConfigTest.java new file mode 100644 index 00000000..213478ee --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswSpringConfigTest.java @@ -0,0 +1,64 @@ +package fr.insalyon.creatis.gasw; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringJUnitConfig(classes = GaswSpringConfigTest.Config.class) +public class GaswSpringConfigTest { + + @Configuration + @EnableScheduling + static class Config { + + @Bean + public ThreadPoolTaskScheduler taskScheduler() { + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setPoolSize(1); + scheduler.setWaitForTasksToCompleteOnShutdown(true); + scheduler.setAwaitTerminationSeconds(30); + scheduler.initialize(); + return scheduler; + } + } + + @Autowired + private ThreadPoolTaskScheduler taskScheduler; + + @Nested + class TaskScheduler { + + @Test + void destroy_waitsForInFlightTaskToComplete() throws Exception { + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch taskFinished = new CountDownLatch(1); + + taskScheduler.execute(() -> { + taskStarted.countDown(); + try { + Thread.sleep(300); + } catch (InterruptedException ignored) { + } finally { + taskFinished.countDown(); + } + }); + assertTrue(taskStarted.await(1, TimeUnit.SECONDS)); + + taskScheduler.destroy(); + + assertEquals(0, taskFinished.getCount(), + "task should have completed before destroy() returned"); + } + } +} \ No newline at end of file diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswTest.java new file mode 100644 index 00000000..a620e0b1 --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswTest.java @@ -0,0 +1,88 @@ +package fr.insalyon.creatis.gasw; + +import fr.insalyon.creatis.gasw.dao.SEEntryPointsDAO; +import fr.insalyon.creatis.gasw.execution.ExecutorFactory; +import fr.insalyon.creatis.gasw.plugin.ExecutorPlugin; +import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class GaswTest { + + @Mock + private GaswConfiguration config; + + @Mock + private GaswNotification gaswNotification; + + @Mock + private ExecutorFactory executorFactory; + + @Mock + private SEEntryPointsDAO seEntryPointsDAO; + + @Mock + private ExecutorPlugin executorPlugin; + + @Mock + private ListenerPlugin listenerPlugin; + + private Gasw gasw; + + @BeforeEach + void setUp() { + gasw = new Gasw(config, gaswNotification, executorFactory, seEntryPointsDAO, + List.of(executorPlugin), List.of(listenerPlugin)); + } + + @Test + @DisplayName("init() does not load SE entry points when failover is disabled") + void init_skipsSELoadingWhenFailOverDisabled() throws GaswException { + when(config.isFailOverEnabled()).thenReturn(false); + gasw.init(); + verifyNoInteractions(seEntryPointsDAO); + } + + @Nested + @DisplayName("terminate(force) cascades to every owned resource") + class Terminate { + + @Test + @DisplayName("stops notification polling, then terminates every executor and listener plugin") + void terminatesNotificationAndAllPlugins() throws GaswException { + gasw.terminate(true); + verify(gaswNotification).terminate(); + verify(executorPlugin).terminate(true); + verify(listenerPlugin).terminate(); + } + + @Test + @DisplayName("propagates the force flag down to executor plugins, not just its own state") + void propagatesForceFlagToExecutorPlugins() throws GaswException { + gasw.terminate(false); + verify(executorPlugin).terminate(false); + } + } + + @Test + @DisplayName("submit() delegates to whichever executor ExecutorFactory currently resolves") + void submit_delegatesToResolvedExecutor() throws GaswException { + GaswInput input = mock(GaswInput.class); + when(executorFactory.getExecutor()).thenReturn(executorPlugin); + when(executorPlugin.submit(input)).thenReturn("job-1"); + assertEquals("job-1", gasw.submit(input)); + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/GaswUtilTest.java b/src/test/java/fr/insalyon/creatis/gasw/GaswUtilTest.java index ec450170..853f8b60 100644 --- a/src/test/java/fr/insalyon/creatis/gasw/GaswUtilTest.java +++ b/src/test/java/fr/insalyon/creatis/gasw/GaswUtilTest.java @@ -3,19 +3,53 @@ import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; @DisplayName("GaswUtil tests") public class GaswUtilTest { - @Test - @DisplayName("Uri detection") - public void uriDetection() { - assertTrue(GaswUtil.isUri("girder:///control_3DT1.nii")); - assertTrue(GaswUtil.isUri("ssh://example.com/control_3DT1.nii")); - assertTrue(GaswUtil.isUri("girder:/control_3DT1.nii")); - assertFalse(GaswUtil.isUri("girder:control_3DT1.nii")); - assertFalse(GaswUtil.isUri("girder:////control_3DT1.nii")); + @Nested + @DisplayName("isUri()") + class IsUri { + + @ParameterizedTest(name = "\"{0}\" is a valid URI") + @DisplayName("recognizes valid URIs with various schemes") + @ValueSource(strings = { + "girder:///control_3DT1.nii", + "shanoir:/control_3DT1.nii", + "http://example.com/file.txt", + "https://example.com/file.txt", + "file:///path/to/file" + }) + void recognizesValidUris(String uri) { + assertTrue(GaswUtil.isUri(uri)); + } + + @ParameterizedTest(name = "\"{0}\" is not a valid URI") + @DisplayName("rejects invalid URIs") + @ValueSource(strings = { + "girder:control_3DT1.nii", + "girder:////control_3DT1.nii", + "/local/path/file.txt", + "file.txt", + "", + "://path" + }) + void rejectsInvalidUris(String uri) { + assertFalse(GaswUtil.isUri(uri)); + } } -} + @Test + @DisplayName("extracts base name from files with extensions") + void extractsBaseName() { + assertEquals("file", GaswUtil.getBaseName("file.txt")); + assertEquals("archive.tar", GaswUtil.getBaseName("archive.tar.gz")); + assertEquals("file", GaswUtil.getBaseName("file")); + assertEquals("", GaswUtil.getBaseName(".hidden")); + assertEquals(".hidden", GaswUtil.getBaseName(".hidden.txt")); + } +} \ No newline at end of file diff --git a/src/test/java/fr/insalyon/creatis/gasw/execution/GaswMonitorTest.java b/src/test/java/fr/insalyon/creatis/gasw/execution/GaswMonitorTest.java new file mode 100644 index 00000000..30b2d20f --- /dev/null +++ b/src/test/java/fr/insalyon/creatis/gasw/execution/GaswMonitorTest.java @@ -0,0 +1,218 @@ +package fr.insalyon.creatis.gasw.execution; + +import fr.insalyon.creatis.gasw.GaswConfiguration; +import fr.insalyon.creatis.gasw.bean.Job; +import fr.insalyon.creatis.gasw.dao.DAOException; +import fr.insalyon.creatis.gasw.dao.JobDAO; +import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +public class GaswMonitorTest { + + public static class TestMonitor extends GaswMonitor { + + public final List replicated = new ArrayList<>(); + public final List killed = new ArrayList<>(); + public final List resumed = new ArrayList<>(); + public final List rescheduled = new ArrayList<>(); + + public TestMonitor(GaswConfiguration config, JobDAO jobDAO, List listeners) { + super(config, jobDAO, listeners); + } + + @Override + public void start() { + } + + @Override + public void terminate() { + } + + @Override + public void add(String jobID, String symbolicName, String fileName, String parameters) { + + } + + @Override + protected void kill(Job job) { + killed.add(job); + } + + @Override + protected void reschedule(Job job) { + rescheduled.add(job); + } + + @Override + protected void replicate(Job job) { + replicated.add(job); + } + + @Override + protected void killReplicas(Job job) { + } + + @Override + protected void resume(Job job) { + resumed.add(job); + } + } + + @Mock + private GaswConfiguration config; + + @Mock + private JobDAO jobDAO; + + @Mock + private ListenerPlugin listenerPlugin; + + private TestMonitor monitor; + + @BeforeEach + void setUp() throws Exception { + Field f = GaswMonitor.class.getDeclaredField("INVOCATION_ID"); + f.setAccessible(true); + AtomicInteger ai = (AtomicInteger) f.get(null); + ai.set(1); + + monitor = new TestMonitor(config, jobDAO, List.of(listenerPlugin)); + } + + @Nested + @DisplayName("add(Job)") + class Add { + + @Test + @DisplayName("assigns a fresh invocation ID and notifies listeners for a new file name") + void newFileName_assignsFreshIdAndNotifies() throws Exception { + Job job = new Job(); + job.setFileName("cmd-42"); + when(jobDAO.getByFileName("cmd-42")).thenReturn(Collections.emptyList()); + + long before = System.currentTimeMillis(); + monitor.add(job); + long after = System.currentTimeMillis(); + + assertTrue(job.getInvocationID() > 0); + assertNotNull(job.getCreation()); + assertTrue(before <= job.getCreation().getTime() && job.getCreation().getTime() <= after, + "creation timestamp should be set to a recent time"); + verify(jobDAO).add(job); + verify(listenerPlugin).jobSubmitted(job); + } + + @Test + @DisplayName("reuses the invocation ID of an existing job sharing the same file name (replicas)") + void existingFileName_reusesInvocationId() throws Exception { + Job existing = new Job(); + existing.setInvocationID(100); + Job job = new Job(); + job.setFileName("cmd-42"); + when(jobDAO.getByFileName("cmd-42")).thenReturn(List.of(existing)); + + monitor.add(job); + assertEquals(100, job.getInvocationID()); + } + } + + @Test + @DisplayName("updateStatus() notifies all listeners before persisting the job") + void updateStatus_notifiesListenersThenPersists() throws Exception { + Job job = new Job(); + + InOrder inOrder = inOrder(listenerPlugin, jobDAO); + monitor.updateStatus(job); + inOrder.verify(listenerPlugin).jobStatusChanged(job); + inOrder.verify(jobDAO).update(job); + } + + @ParameterizedTest(name = "completedJobsByInvocationId={0} -> isReplica={1}") + @CsvSource({ + "1, true", + "0, false" + }) + @DisplayName("returns true only when another job with the same invocation ID has already completed") + void isReplica_returnsExpectedResult(long completedJobs, boolean expected) throws Exception { + Job job = new Job(); + job.setInvocationID(5); + + when(jobDAO.getNumberOfCompletedJobsByInvocationID(5)).thenReturn(completedJobs); + + assertEquals(expected, monitor.isReplica(job)); + } + + @Nested + @DisplayName("verifySignaledJobs() status-based dispatch") + class VerifySignaledJobs { + + @ParameterizedTest(name = "{0} -> dispatch") + @CsvSource({ + "REPLICATE, REPLICATED", + "KILL, KILLED", + "KILL_REPLICA, KILLED" + }) + void dispatchesJobs(GaswStatus status, String expectedJobList) throws Exception { + Job job = new Job(); + + when(jobDAO.getJobs(status)).thenReturn(List.of(job)); + stubEmptyExcept(status); + + monitor.verifySignaledJobs(); + + switch (expectedJobList) { + case "REPLICATED" -> assertTrue(monitor.replicated.contains(job)); + case "KILLED" -> assertTrue(monitor.killed.contains(job)); + default -> fail("Unknown job list: " + expectedJobList); + } + } + + @ParameterizedTest(name = "{0} -> {1}") + @CsvSource({ + "UNHOLD_ERROR, ERROR", + "UNHOLD_STALLED, STALLED" + }) + void resumesUnholdJobs(GaswStatus status, GaswStatus expectedStatus) throws Exception { + Job job = new Job(); + + when(jobDAO.getJobs(status)).thenReturn(List.of(job)); + stubEmptyExcept(status); + + monitor.verifySignaledJobs(); + + assertEquals(expectedStatus, job.getStatus()); + assertTrue(monitor.resumed.contains(job)); + verify(jobDAO).update(job); + } + + private void stubEmptyExcept(GaswStatus except) throws DAOException { + for (GaswStatus status : List.of(GaswStatus.REPLICATE, GaswStatus.KILL_REPLICA, GaswStatus.KILL, + GaswStatus.RESCHEDULE, GaswStatus.UNHOLD_ERROR, GaswStatus.UNHOLD_STALLED)) { + if (status != except) { + when(jobDAO.getJobs(status)).thenReturn(Collections.emptyList()); + } + } + } + } +} diff --git a/src/test/java/fr/insalyon/creatis/gasw/parser/GaswParserTest.java b/src/test/java/fr/insalyon/creatis/gasw/parser/GaswParserTest.java index 9accf028..26f85d05 100644 --- a/src/test/java/fr/insalyon/creatis/gasw/parser/GaswParserTest.java +++ b/src/test/java/fr/insalyon/creatis/gasw/parser/GaswParserTest.java @@ -6,43 +6,29 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @DisplayName("GaswParser URI handling tests") class GaswParserTest { - @Test - @DisplayName("URI with 3 slashes") - public void uriGetPathGetNameTripleSlash() throws URISyntaxException { - String value = "girder:///control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69"; - - URI valueURI = new URI(value); - String res = new File(valueURI.getPath()).getName(); - - assertEquals("control_3DT1.nii", res); - } - - @Test - @DisplayName("URI with 1 slash") - public void uriGetPathGetNameSingleSlash() throws URISyntaxException { - String value = "girder:/control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69"; + @ParameterizedTest + @DisplayName("URI test with different number of slashes") + @CsvSource({ + "girder:///control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69, control_3DT1.nii", + "girder:/control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69, control_3DT1.nii", + "girder://control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69, ''" + }) + void uriPathExtraction(String value, String expected) throws URISyntaxException { + URI uri = new URI(value); - URI valueURI = new URI(value); - String res = new File(valueURI.getPath()).getName(); - - assertEquals("control_3DT1.nii", res); - } - - @Test - @DisplayName("URI with 2 slashes has no file") - public void uriGetPathGetNameDouleSlash() throws URISyntaxException { - String value = "girder://control_3DT1.nii?apiurl=http://localhost:8080/api/v1&fileId=5ae1a8fc371210092e0d2936&token=TFT2FdxP9hzM7WKsidBjMJMmN69"; - - URI valueURI = new URI(value); - String res = new File(valueURI.getPath()).getName(); + String path = uri.getPath(); + String result = (path == null) ? "" : new java.io.File(path).getName(); - assertEquals("", res); + assertEquals(expected, result); } @Test diff --git a/src/test/java/fr/insalyon/creatis/gasw/parser/output/DumpOutputParser.java b/src/test/java/fr/insalyon/creatis/gasw/parser/output/DumpOutputParser.java index 48e00d3d..4908ec06 100644 --- a/src/test/java/fr/insalyon/creatis/gasw/parser/output/DumpOutputParser.java +++ b/src/test/java/fr/insalyon/creatis/gasw/parser/output/DumpOutputParser.java @@ -1,26 +1,40 @@ package fr.insalyon.creatis.gasw.parser.output; import java.io.File; +import java.io.IOException; +import java.util.List; +import fr.insalyon.creatis.gasw.GaswConfiguration; import fr.insalyon.creatis.gasw.GaswException; +import fr.insalyon.creatis.gasw.GaswNotification; import fr.insalyon.creatis.gasw.GaswOutput; +import fr.insalyon.creatis.gasw.dao.DataDAO; +import fr.insalyon.creatis.gasw.dao.JobDAO; +import fr.insalyon.creatis.gasw.dao.JobMinorStatusDAO; +import fr.insalyon.creatis.gasw.dao.NodeDAO; import fr.insalyon.creatis.gasw.execution.GaswOutputParser; +import fr.insalyon.creatis.gasw.execution.GaswParsingContext; +import fr.insalyon.creatis.gasw.plugin.ListenerPlugin; +import org.springframework.stereotype.Service; +@Service public class DumpOutputParser extends GaswOutputParser { - public DumpOutputParser(String jobID) { - super(jobID); + public DumpOutputParser(GaswConfiguration config, GaswNotification gaswNotification, + JobDAO jobDAO, JobMinorStatusDAO jobMinorStatusDAO, NodeDAO nodeDAO, + DataDAO dataDAO, List listenerPlugins) { + super(config, gaswNotification, jobDAO, jobMinorStatusDAO, nodeDAO, dataDAO, listenerPlugins); } @Override - public GaswOutput getGaswOutput() throws GaswException { + public GaswOutput getGaswOutput(GaswParsingContext context) throws GaswException { return null; } @Override protected void resubmit() throws GaswException {} - public void parseStdout(File file) throws GaswException { - parseStdOut(file); + public int parseStdout(File file, GaswParsingContext context) throws IOException { + return parseStdOut(file, context); } } diff --git a/src/test/java/fr/insalyon/creatis/gasw/parser/output/GaswOutputParserTest.java b/src/test/java/fr/insalyon/creatis/gasw/parser/output/GaswOutputParserTest.java index fd8d73b7..d957a96a 100644 --- a/src/test/java/fr/insalyon/creatis/gasw/parser/output/GaswOutputParserTest.java +++ b/src/test/java/fr/insalyon/creatis/gasw/parser/output/GaswOutputParserTest.java @@ -1,14 +1,6 @@ package fr.insalyon.creatis.gasw.parser.output; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; - import java.io.File; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -19,105 +11,86 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import ch.qos.logback.classic.Logger; +import fr.insalyon.creatis.gasw.dao.DAOException; +import fr.insalyon.creatis.gasw.dao.JobDAO; +import fr.insalyon.creatis.gasw.execution.GaswParsingContext; +import fr.insalyon.creatis.gasw.bean.Job; +import fr.insalyon.creatis.gasw.execution.GaswStatus; + +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Logger; - -import org.h2.jdbcx.JdbcDataSource; - -import fr.insalyon.creatis.gasw.GaswConfiguration; -import fr.insalyon.creatis.gasw.GaswException; -import fr.insalyon.creatis.gasw.bean.Job; -import fr.insalyon.creatis.gasw.dao.DAOException; -import fr.insalyon.creatis.gasw.dao.DAOFactory; -import fr.insalyon.creatis.gasw.execution.GaswStatus; -import fr.insalyon.creatis.gasw.plugin.DatabasePlugin; +import static org.junit.jupiter.api.Assertions.*; +@ExtendWith(MockitoExtension.class) public class GaswOutputParserTest { - private static final Logger logger = (Logger) LoggerFactory.getLogger(GaswOutputParserTest.class); + private final Logger logger = (Logger) LoggerFactory.getLogger(getClass()); + + @Mock + private JobDAO jobData; @Mock - private DatabasePlugin databasePlugin; + private DumpOutputParser dumpOutputParser; - private GaswConfiguration config; private MemoryAppender appender; @BeforeEach - public void mockDB() throws GaswException, SQLException { - GaswConfiguration.setStrict(false); - config = GaswConfiguration.getInstance(); - MockitoAnnotations.openMocks(this); - - when(databasePlugin.getConnectionUrl()).thenReturn("jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=TRUE"); - when(databasePlugin.getDriverClass()).thenReturn("org.h2.Driver"); - when(databasePlugin.getHibernateDialect()).thenReturn("org.hibernate.dialect.H2Dialect"); - when(databasePlugin.getName()).thenReturn("test"); - when(databasePlugin.getPassword()).thenReturn("pass"); - when(databasePlugin.getSchema()).thenReturn("test"); - when(databasePlugin.getUserName()).thenReturn("test"); - - JdbcDataSource source = new JdbcDataSource(); - source.setPassword(databasePlugin.getPassword()); - source.setUser(databasePlugin.getUserName()); - source.setUrl(databasePlugin.getConnectionUrl()); - - try (Connection connection = source.getConnection()) { - try (Statement stmt = connection.createStatement()) { - stmt.execute("CREATE SCHEMA IF NOT EXISTS " + databasePlugin.getSchema()); - } - } + public void configureAppender() { + appender = new MemoryAppender(); + logger.addAppender(appender); + appender.start(); + } - config.setDbPlugin(databasePlugin); - config.loadHibernate(); - assertTrue(config.getSessionFactory() != null); + @AfterEach + public void cleanupAppender() { + logger.detachAppender(appender); + appender.stop(); } /** * For this test, we try to stress the system by creating multi-threads output parser at the same time, * this ensure the realiability of the parser and also hibernate. - * The custom appender is used to capture logger.error used in some function instead + * The custom appender is used to capture logger.error used in some function instead * of catching exception (because they are catched in sublayers and not rethrown) */ @Test + @DisplayName("concurrent parsing of same output file by multiple threads") public void testMultiOutputSameTime() throws DAOException, InterruptedException, ExecutionException { int tSize = 10; ExecutorService service = Executors.newFixedThreadPool(tSize); List> callables = new ArrayList<>(); List> parsers = new ArrayList<>(); - Job job = new Job("test", "test_sim", GaswStatus.CREATED, "echo", "coucou", "a,b,c", "Local"); + Job job = new Job("test", "test_sim", GaswStatus.CREATED, "echo", "test-job.sh", "a,b,c", "Local"); job.setDownload(new Date()); - - DAOFactory.getDAOFactory().getJobDAO().add(job); + jobData.add(job); for (int i = 0; i < tSize; i++) { - callables.add(createCallable("test", "src/test/resources/execA.out")); + callables.add(createCallable(job, "src/test/resources/execA.out")); } - configureAppender(); parsers = service.invokeAll(callables); + service.shutdown(); + assertTrue(service.awaitTermination(20, TimeUnit.SECONDS)); for (Future parser : parsers) { assertDoesNotThrow(() -> parser.get(10, TimeUnit.SECONDS)); } - assertFalse(appender.getLogMessages().stream().anyMatch(msg -> msg.contains("Error parsing stdout"))); + assertFalse(appender.getLogMessages().stream().anyMatch(msg -> msg.contains("Error parsing stdout"))); } - public Callable createCallable(String jobID, String filePath) { + private Callable createCallable(Job job, String filePath) { return () -> { - DumpOutputParser parser = new DumpOutputParser(jobID); - - parser.parseStdout(new File(filePath)); + GaswParsingContext context = new GaswParsingContext(job); + dumpOutputParser.parseStdout(new File(filePath), context); return null; }; } - - public void configureAppender() { - appender = new MemoryAppender(); - logger.addAppender(appender); - } } diff --git a/src/test/java/fr/insalyon/creatis/gasw/script/DataManagementGeneratorTest.java b/src/test/java/fr/insalyon/creatis/gasw/script/DataManagementGeneratorTest.java deleted file mode 100644 index 689b5d7c..00000000 --- a/src/test/java/fr/insalyon/creatis/gasw/script/DataManagementGeneratorTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* Copyright CNRS-CREATIS - * - * Rafael Ferreira da Silva - * rafael.silva@creatis.insa-lyon.fr - * http://www.rafaelsilva.com - * - * This software is governed by the CeCILL license under French law and - * abiding by the rules of distribution of free software. You can use, - * modify and/ or redistribute the software under the terms of the CeCILL - * license as circulated by CEA, CNRS and INRIA at the following URL - * "http://www.cecill.info". - * - * As a counterpart to the access to the source code and rights to copy, - * modify and redistribute granted by the license, users are provided only - * with a limited warranty and the software's author, the holder of the - * economic rights, and the successive licensors have only limited - * liability. - * - * In this respect, the user's attention is drawn to the risks associated - * with loading, using, modifying and/or developing or reproducing the - * software by the user in light of its specific status of free software, - * that may mean that it is complicated to manipulate, and that also - * therefore means that it is reserved for developers and experienced - * professionals having in-depth computer knowledge. Users are therefore - * encouraged to load and test the software's suitability as regards their - * requirements in conditions enabling the security of their systems and/or - * data to be ensured and, more generally, to use and operate it in the - * same conditions as regards security. - * - * The fact that you are presently reading this means that you have had - * knowledge of the CeCILL license and that you accept its terms. - */ -package fr.insalyon.creatis.gasw.script; - -import fr.insalyon.creatis.gasw.util.VelocityUtil; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -@DisplayName("Check velocity templates") -class DataManagementGeneratorTest { - - @Test - @DisplayName("Check checkCacheDownloadAndCacheLFNFunction template") - public void checkCheckCacheDownloadAndCacheLFNFunctionTemplate() - throws Exception { - - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/checkCacheDownloadAndCacheLFNFunction.vm", false); - - velocity.put("cacheDir", "dir"); - velocity.put("cacheFile", "file"); - - velocity.merge().toString(); - } - - @Test - @DisplayName("Check variables template") - public void checkVariablesTemplateNoVariable() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/execution/variables.vm", false); - - velocity.put("variables", new HashMap<>()); - - String res = velocity.merge().toString(); - - Assertions.assertFalse(res.contains("export")); - } - - @Test - @DisplayName("Check variables template") - public void checkVariablesTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/execution/variables.vm", false); - - Map variables = new HashMap<>() {{ - put("var1", "value1"); - put("var2", "value2"); - }}; - velocity.put("variables", variables); - - String res = velocity.merge().toString(); - - Assertions.assertEquals(2, res.lines().filter(l -> l.contains("export")).count()); - Assertions.assertTrue(res.contains("export var1=\"value1\"")); - Assertions.assertTrue(res.contains("export var2=\"value2\"")); - } - - @Test - @DisplayName("Check downloadFunction template") - public void checkDownloadFunctionTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/downloadFunctions.vm", false); - - velocity.put("timeout", 10); - velocity.put("minAvgDownloadThroughput", 10); - velocity.put("bdiiTimeout", 10); - velocity.put("srmTimeout", 10); - velocity.put("failOverEnabled", false); - velocity.put("failOverHost", "host"); - velocity.put("failOverPort", 10); - velocity.put("failOverHome", "home"); - - velocity.merge().toString(); - } - - @Test - @DisplayName("Check addToCacheFunction template") - public void checkAddToCacheFunctionTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/addToCacheFunction.vm", false); - - velocity.put("cacheDir", "dir"); - velocity.put("cacheFile", "file"); - - velocity.merge().toString(); - } - - @Test - @DisplayName("Check addToFailOverFunction template") - public void checkAddToFailOverFunctionTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/addToFailOverFunction.vm", false); - - velocity.put("failOverHost", "host"); - velocity.put("failOverPort", 10); - velocity.put("failOverHome", "home"); - - velocity.merge().toString(); - } - - @Test - @DisplayName("Check uploadFunction template") - public void checkUploadFunctionTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/uploadFunctions.vm", false); - - velocity.put("timeout", 10); - velocity.put("minAvgDownloadThroughput", 10); - velocity.put("bdiiTimeout", 10); - velocity.put("srmTimeout", 10); - velocity.put("failOverEnabled", false); - - velocity.merge().toString(); - } - - @Test - @DisplayName("Check deleteFunctions template") - public void checkDeleteFunctionsTemplate() throws Exception { - VelocityUtil velocity = new VelocityUtil("vm/script/datamanagement/deleteFunctions.vm", false); - - velocity.merge().toString(); - } -} diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties new file mode 100644 index 00000000..a044ff87 --- /dev/null +++ b/src/test/resources/application-test.properties @@ -0,0 +1,31 @@ +gasw.default.executor=local +gasw.default.environment=test +gasw.default.background-script=test.sh +gasw.default.requirements= +gasw.default.retry-count=3 +gasw.default.timeout=3600 +gasw.default.sleep-time=5 +gasw.default.cpu-time=3600 +gasw.vo.name=test +gasw.vo.default-SE=test-se +gasw.vo.use-close-SE=false +gasw.boutiques.bosh-CVMFS-path=/cvmfs/test +gasw.boutiques.file-name=test.json +gasw.boutiques.provenance-dir=/tmp/provenance +gasw.containers.runtime=docker +gasw.containers.images-base-path=/tmp/images +gasw.containers.singularity-path=/usr/bin/singularity +gasw.containers.CVMFS-path=/cvmfs/containers +gasw.containers.udocker-tag=1.0 +gasw.failover.enabled=false +gasw.failover.host=localhost +gasw.failover.port=8080 +gasw.failover.home=/tmp/failover +gasw.failover.max-retry=3 +gasw.download.min-avg-throughput=150 +gasw.minor-status.enabled=false +gasw.source.script= +gasw.scheduler.pool-size=1 +gasw.scheduler.thread-name-prefix=test-scheduler- +gasw.scheduler.await-termination-seconds=30 +gasw.scheduler.wait-for-tasks-on-shutdown=true \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index ef5d4a01..601a5e64 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -46,4 +46,6 @@ + +