I'm trying to run a sample of akka-quartz-scheduler, I got the akka-quickstart-java, and created a new main class for the scheduler sample, then I added the application.conf (under main/resources) and I added the dependencies to build.sbt.
I tried to load manually the Config with: ConfigFactory.load(), I got the content of the config file correctly, but it appears that the file is never seen by the schedulerExtension.
SchedulerQuickstart.java
public class SchedulerQuickstart {
public static void main(String[] args){
final ActorSystem system = ActorSystem.create("helloscheduler");
try {
QuartzSchedulerExtension schedulerExtension = new QuartzSchedulerExtension((ExtendedActorSystem) system);
Date firstExecutionDate=schedulerExtension.schedule("GreetingSchedule",
system.actorOf(Printer.props(), "printerActor"),
new Printer.Greeting("This is an scheduled message"));
}
catch (Exception ex){
}
finally {
system.terminate();
}
}
}
application.conf
akka.quartz.schedules {
GreetingSchedule {
description = "Task that fires off every 30 seconds"
expression = "*/30 * * ? * *"
maxRetries = 2
}
}
in build.sbt:
"com.enragedginger" %% "akka-quartz-scheduler" % "1.8.0-akka-2.5.x"
When I run the code, I got:
No matching quartz configuration found for schedule 'GreetingSchedule'
Related
This is for embed jetty. I am trying to clean the jetty working directory which is automatically created in the /tmp folder inside the container. I did write the below method-"cleanJettyWorkingDirectory()" logic to clean the working directory and it works. The problem here is, it is not allowing me to create a working directory now because think I am calling this method from the wrong place. Whenever I am restarting the docker, it is cleaning the entire working directory Please assist.
public void cleanJettyWorkingDirectory(){
final File folder = new File(JETTY_WORKING_DIRECTORY);
final File[] files = folder.listFiles( new FilenameFilter() {
#Override
public boolean accept( final File dir,
final String name ) {
return name.matches( "jetty-0_0_0_0-.*" );
}
} );
for ( final File file : files ) {
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
logger.info("Unable to delete the Jetty working directory");
}
}
}
Jetty Service Main class file as below:
public class JettyServer {
private final static Logger logger = Logger.getLogger(JettyServer.class.getName());
private static final int JETTY_PORT = 10000;
private static final String JETTY_REALM_PROPERTIES_FILE_NAME = "realm.properties";
private static final String JETTY_REALM_NAME = "myrealm";
private static final String JETTY_WORKING_DIRECTORY="tmp";
public static QueuedThreadPool threadPool;
public JettyServer() {
try {
cleanJettyWorkingDirectory(); // *Calling here*
RolloverFileOutputStream os = new RolloverFileOutputStream(JETTY_STDOUT_LOG_FILE_NAME, true);
PrintStream logStream = new PrintStream(os);
System.setOut(logStream);
System.setErr(logStream);
Server server = new Server(JETTY_PORT);
server.addBean(getLoginService());
try {
logger.info("Configuring Jetty SSL..");
HttpConfiguration http_config = new HttpConfiguration();
http_config.setSecureScheme("https");
http_config.setSecurePort(JETTY_PORT);
https.setPort(JETTY_PORT);
server.setConnectors(new Connector[]{https});
logger.info("Jetty SSL successfully configured..");
} catch (Exception e){
logger.severe("Error configuring Jetty SSL.."+e);
throw e;
}
Configuration.ClassList classlist = Configuration.ClassList.setServerDefault(server);
classlist.addAfter("org.eclipse.jetty.webapp.FragmentConfiguration",
"org.eclipse.jetty.plus.webapp.EnvConfiguration",
"org.eclipse.jetty.plus.webapp.PlusConfiguration");
//register ui and service web apps
HandlerCollection webAppHandlers = getWebAppHandlers();
for (Connector c : server.getConnectors()) {
c.getConnectionFactory(HttpConnectionFactory.class).getHttpConfiguration().setRequestHeaderSize(MAX_REQUEST_HEADER_SIZE);
c.getConnectionFactory(HttpConnectionFactory.class).getHttpConfiguration().setSendServerVersion(false);
}
threadPool = (QueuedThreadPool) server.getThreadPool();
// request logs
RequestLogHandler requestLogHandler = new RequestLogHandler();
AsyncRequestLogWriter asyncRequestLogWriter = new AsyncRequestLogWriter(JETTY_REQUEST_LOG_FILE_NAME);
asyncRequestLogWriter.setFilenameDateFormat(JETTY_REQUEST_LOG_FILE_NAME_DATE_FORMAT);
asyncRequestLogWriter.setAppend(JETTY_REQUEST_LOG_FILE_APPEND);
asyncRequestLogWriter.setRetainDays(JETTY_REQUEST_LOG_FILE_RETAIN_DAYS);
asyncRequestLogWriter.setTimeZone(TimeZone.getDefault().getID());
requestLogHandler.setRequestLog(new AppShellCustomRequestLog(asyncRequestLogWriter));
webAppHandlers.addHandler(requestLogHandler);
StatisticsHandler statisticsHandler = new StatisticsHandler();
statisticsHandler.setHandler(new AppshellStatisticsHandler());
webAppHandlers.addHandler(statisticsHandler);
// set handler
server.setHandler(webAppHandlers);
//start jettyMetricsPsr
JettyMetricStatistics.logJettyMetrics();
// set error handler
server.addBean(new CustomErrorHandler());
// GZip Handler
GzipHandler gzip = new GzipHandler();
server.setHandler(gzip);
gzip.setHandler(webAppHandlers);
//setting server attribute for datasources
server.setAttribute("fawappshellDS", new Resource(JNDI_NAME_FAWAPPSHELL, DatasourceUtil.getFawAppshellDatasource()));
server.setAttribute("fawcommonDS", new Resource(JNDI_NAME_FAWCOMMON, DatasourceUtil.getCommonDatasource()));
//new Resource(server, JNDI_NAME_FAWAPPSHELL, getFawAppshellDatasource());
//new Resource(server, JNDI_NAME_FAWCOMMON, getFawCommonDatasource());
server.start();
server.join();
} catch (Exception e) {
e.printStackTrace();
}
}
private HandlerCollection getWebAppHandlers() throws SQLException, NamingException{
//Setting the war and context path for the service layer: oaxservice
WebAppContext serviceWebapp = new WebAppContext();
serviceWebapp.setWar(APPSHELL_API_WAR_FILE_PATH);
serviceWebapp.setContextPath(APPSHELL_API_CONTEXT_PATH);
serviceWebapp.setPersistTempDirectory(false);
//setting the war and context path for the UI layer: oaxui
WebAppContext uiWebapp = new WebAppContext();
uiWebapp.setWar(APPSHELL_UI_WAR_FILE_PATH);
uiWebapp.setContextPath(APPSHELL_UI_CONTEXT_PATH);
uiWebapp.setAllowNullPathInfo(true);
uiWebapp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
//set error page handler for the UI context
uiWebapp.setErrorHandler(new CustomErrorHandler());
//handling the multiple war files using HandlerCollection.
HandlerCollection handlerCollection = new HandlerCollection();
handlerCollection.setHandlers(new Handler[]{serviceWebapp, uiWebapp});
return handlerCollection;
}
public LoginService getLoginService() throws IOException {
URL realmProps = JettyServer.class.getClassLoader().getResource(JETTY_REALM_PROPERTIES_FILE_NAME);
if (realmProps == null)
throw new FileNotFoundException("Unable to find " + JETTY_REALM_PROPERTIES_FILE_NAME);
return new HashLoginService(JETTY_REALM_NAME, realmProps.toExternalForm());
}
public void cleanJettyWorkingDirectory(){
final File folder = new File(JETTY_WORKING_DIRECTORY);
final File[] files = folder.listFiles( new FilenameFilter() {
#Override
public boolean accept( final File dir,
final String name ) {
return name.matches( "jetty-0_0_0_0-.*" );
}
} );
for ( final File file : files ) {
try {
FileUtils.deleteDirectory(file);
} catch (IOException e) {
logger.info("Unable to delete the Jetty working directory");
}
}
}
public static void main(String[] args) {
new JettyServer();
}
}
Option 1: Use docker tmpfs
If you want to eliminate the system temp persistence, just use docker correctly to avoid it doing that between restarts, don't write this custom logic within your java app.
The docker tmpfs is probably going to be a better solution.
See past answer: https://stackoverflow.com/a/52662602/775715
Option 2: Use linux systemd tmpfiles
You could also use systemd-tmpfiles or systemd-tmpfiles-clean to perform the cleanup (periodically) automatically within the Linux environment within your docker image.
Option 3: Use a non-standard system temp directory for Jetty
Configure a new Temp Directory for your Java instance ...
$ java -Djava.io.tmpdir=/var/run/jetty/work/ -jar start.jar
Then use your shell script that starts your Jetty instance to just clear out that unique directory before you execute the java instance.
aka:
JETTY_WORK=/var/run/jetty/work
rm -rf $JETTY_WORK/*
java -Djava.io.tmpdir=$JETTY_WORK/ -jar start.jar
This approach also catches all Java temp directory usages from your 3rd party libraries as well, not just Jetty itself.
Is it possible to run Pentaho ETL Jobs/transformation using AWS Lamda functions?
I have Pentaho ETL jobs running on schedule on the Windows server, we are planning to migrate to AWS. I am considering the Lambda function. just to understand if it is possible to schedule the Pentaho ETL Jobs using AWS Lamdba
Here is the snippet of code that I was able to successfully run in AWS Lambda Function.
handleRequest Function is called from AWS Lambda Function
public Integer handleRequest(String input, Context context) {
parseInput(input);
return executeKtr(transName);
}
parseInput: This function is used to parse out a string parameter passed by Lambda Function to extract KTR name and its parameters with value. Format of the input is "ktrfilename param1=value1 param2=value2"
public static void parseInput(String input) {
String[] tokens = input.split(" ");
transName = tokens[0].replace(".ktr", "") + ".ktr";
for (int i=1; i<tokens.length; i++) {
params.add(tokens[i]);
}
}
Executing KTR: I am using git repo to store all my KTR files and based on the name passed as a parameter KTR is executed
public static Integer executeKtr(String ktrName) {
try {
System.out.println("Present Project Directory : " + System.getProperty("user.dir"));
String transName = ktrName.replace(".ktr", "") + ".ktr";
String gitURI = awsSSM.getParaValue("kattle-trans-git-url");
String repoLocalPath = clonePDIrepo.cloneRepo(gitURI);
String path = new File(repoLocalPath + "/" + transName).getAbsolutePath();
File ktrFile = new File(path);
System.out.println("KTR Path: " + path);
try {
/**
* IMPORTANT NOTE FOR LAMBDA FUNCTION MUST CREATE .KEETLE DIRECOTRY OTHERWISE
* CODE WILL FAIL IN LAMBDA FUNCTION WITH ERROR CANT CREATE
* .kettle/kettle.properties file.
*
* ALSO SET ENVIRNOMENT VARIABLE ON LAMBDA FUNCTION TO POINT
* KETTLE_HOME=/tmp/.kettle
*/
Files.createDirectories(Paths.get("/tmp/.kettle"));
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("Error Creating /tmp/.kettle directory");
}
if (ktrFile.exists()) {
KettleEnvironment.init();
TransMeta metaData = new TransMeta(path);
Trans trans = new Trans(metaData);
// SETTING PARAMETERS
trans = parameterSetting(trans);
trans.execute( null );
trans.waitUntilFinished();
if (trans.getErrors() > 0) {
System.out.print("Error Executing transformation");
throw new RuntimeException("There are errors in running transformations");
} else {
System.out.print("Successfully Executed Transformation");
return 1;
}
} else {
System.out.print("KTR File:" + path + " not found in repo");
throw new RuntimeException("KTR File:" + path + " not found in repo");
}
} catch (KettleException e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
}
parameterSetting: If KTR is accepting parameter and it is passed while calling AWS Lambda function, it is set using parameterSetting function.
public static Trans parameterSetting(Trans trans) {
String[] transParams = trans.listParameters();
for (String param : transParams) {
for (String p: params) {
String name = p.split("=")[0];
String val = p.split("=")[1];
if (name.trim().equals(param.trim())) {
try {
System.out.println("Setting Parameter:"+ name + "=" + val);
trans.setParameterValue(name, val);
} catch (UnknownParamException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
trans.activateParameters();
return trans;
}
CloneGitRepo:
public class clonePDIrepo {
/**
* Clones the given repo to local folder
*
* #param pathWithPwd Gir repo URL with access token included in the url. e.g.
* https://token_name:token_value#github.com/ktr-git-repo.git
* #return returns Local Repository String Path
*/
public static String cloneRepo(String pathWithPwd) {
try {
/**
* CREATING TEMP DIR TO AVOID FOLDER EXISTS ERROR, THIS TEMP DIRECTORY LATER CAN
* BE USED TO GET ABSOLETE PATH FOR FILES IN DIRECTORY
*/
File pdiLocalPath = Files.createTempDirectory("repodir").toFile();
Git git = Git.cloneRepository().setURI(pathWithPwd).setDirectory(pdiLocalPath).call();
System.out.println("Git repository cloned successfully");
System.out.println("Local Repository Path:" + pdiLocalPath.getAbsolutePath());
// }
return pdiLocalPath.getAbsolutePath();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
}
AWSSSMgetParaValue: Gets string value of the parameter passed.
public static String getParaValue(String paraName) {
try {
Region region = Region.US_EAST_1;
SsmClient ssmClient = SsmClient.builder()
.region(region)
.build();
GetParameterRequest parameterRequest = GetParameterRequest.builder()
.name(paraName)
.withDecryption(true)
.build();
GetParameterResponse parameterResponse = ssmClient.getParameter(parameterRequest);
System.out.println(paraName+ " value retreived from AWS SSM");
ssmClient.close();
return parameterResponse.parameter().value();
} catch (SsmException e) {
System.err.println(e.getMessage());
return null;
}
}
Assumptions:
Git repo is created with KTR files in the root of the repo
git repo url exists on the aws SSM with valid tokens to clone the repo
Input string contains name of the KTR file
Environment Variable is configured on Lambda Function for KETTLE_HOME=/tmp/.kettle
Lambda Function has necessary permissions for SSM and S3 VPC Network
Proper Security Group rules are setup to allow required network access for the KTR File
I am planning to upload complete code to git. I will update this post with the URL of the repository.
Environment: HDP 2.3 Sandbox
Problem: I have created a table in hive with just 2 columns. Now i want to read this in my MR code using HCatalog integration. The MR Job fails to read the table from the MySql meta-store. It uses the Derby for some reason and hence it fails with "table not found" message.
Job Client code:
public class HCatalogMRJob extends Configured implements Tool {
public int run(String[] args) throws Exception {
Configuration conf = getConf();
args = new GenericOptionsParser(conf, args).getRemainingArgs();
String inputTableName = args[0];
String outputTableName = args[1];
String dbName = null;
Job job = new Job(conf, "HCatalogMRJob");
HCatInputFormat.setInput(job, dbName, inputTableName);
job.setInputFormatClass(HCatInputFormat.class);
job.setJarByClass(HCatalogMRJob.class);
job.setMapperClass(HCatalogMapper.class);
job.setReducerClass(HCatalogReducer.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(WritableComparable.class);
job.setOutputValueClass(DefaultHCatRecord.class);
HCatOutputFormat.setOutput(job, OutputJobInfo.create(dbName, outputTableName, null));
HCatSchema s = HCatOutputFormat.getTableSchema(conf);
System.err.println("INFO: output schema explicitly set for writing:"
+ s);
HCatOutputFormat.setSchema(job, s);
job.setOutputFormatClass(HCatOutputFormat.class);
return (job.waitForCompletion(true) ? 0 : 1);
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new HCatalogMRJob(), args);
System.exit(exitCode);
}
}
Job Run Command:
hadoop jar mr-hcat.jar input_table out_table
Before running this command, i have set the necessary hcatalog, hive jars in the class path using the hadoop_classpath variable.
Question:
Now, how do i make the job to use the hive-site.xml correctly?
I tried setting this in the classpath using the same hadoop_classpath as mentioned above., but still it fails.
i have a web service.
it takes 10-15 seconds to upload completely. But some time due to slow internet connection it takes more than 15 second.
I want to open a message box or pop up box which open when the service takes more than 15 seconds, to display message "Slow Internet Connection".
How can i add timer in my code so that a pop up block open(Or alert says Slow Connection) on page load event when web service takes time more than 15 second to load ?
Please help me on this...
public Quiz()
{
InitializeComponent();
DispatchTimer timer = new DispatcherTimer();
int serviceCount = 15;
Loaded += Quiz_Loaded;
}
protected void Quiz_Loaded(object sender, RoutedEventArgs e)
{
SystemTray.ProgressIndicator = new ProgressIndicator();
SystemTray.ProgressIndicator.IsIndeterminate = true;
SystemTray.ProgressIndicator.IsVisible = true;
SystemTray.ProgressIndicator.Text = "Loading Questions...";
pg2.Visibility = Visibility.Visible;
txtloading.Visibility = Visibility.Visible;
if (!timer.IsEnabled)
{
timer.Tick += timer_Tick;
timer.Interval = new TimeSpan(0, 0, 1);
timer.Start();
}
}
void timer_Tick(object sender, object e)
{
serviceCount--;
if (serviceCount < 15)
{
PostData();
}
else
{
txtloading.Text = "slow connection.....";
}
}
private void PostData()
{
Uri uri = new Uri("my web service url");
string data = "device_id=test123&quiz_type=all";
WebClient wc = new WebClient();
wc.Headers[HttpRequestHeader.ContentType] = "application/x-www-form-urlencoded";
wc.UploadStringAsync(uri, data);
wc.UploadStringCompleted += wc_UploadComplete;
}
public void wc_UploadComplete(object sender, UploadStringCompletedEventArgs e)
{
var rootObject = JsonConvert.DeserializeObject<RootObject>(e.Result);
question = rootObject.questions;
DisplayQuestion();
SystemTray.ProgressIndicator.IsVisible = false;
pg2.Visibility = Visibility.Collapsed;
txtloading.Visibility = Visibility.Collapsed;
}
i want to call message block when web service takes time more than 15 second. & also have a retry button which reload the page again to get all data from service..
You could use a DispatcherTimer that is initialized whenever the first call to the web service is initiated. That way, whenever a given timespan elapses, you can trigger a custom action (such as a notification).
This seems like a simple thing to do but I can't seem to find any info anywhere! I've got a solution that has a service that we run in 'Console Mode' when debugging. I want it to be started and 'attached' when I run my unit test from Visual Studio.
I'm using Resharper as the unit test runner.
Not a direct answer to your question, BUT
We faced a similar problem recently and eventually settled on a solution using AppDomain
As your solution is already running as a Console project it would be little work to make it boot in a new AppDomain. Furthermore, you could run Assertions on this project as well as part of unit testing. (if required)
Consider the following static class Sandbox which you can use to boot multiple app domains.
The Execute method requires a Type which is-a SandboxAction. (class definition also included below)
You would first extend this class and provide any bootup actions for running your console project.
public class ConsoleRunnerProjectSandbox : SandboxAction
{
protected override void OnRun()
{
Bootstrapper.Start(); //this code will be run on the newly create app domain
}
}
Now to get your app domain running you simply call
Sandbox.Execute<ConsoleRunnerProjectSandbox>("AppDomainName", configFile)
Note you can pass this call a config file so you can bootup your project in the same fashion as if you were running it via the console
Any more questions please ask.
public static class Sandbox
{
private static readonly List<Tuple<AppDomain, SandboxAction>> _sandboxes = new List<Tuple<AppDomain, SandboxAction>>();
public static T Execute<T>(string friendlyName, string configFile, params object[] args)
where T : SandboxAction
{
Trace.WriteLine(string.Format("Sandboxing {0}: {1}", typeof (T).Name, configFile));
AppDomain sandbox = CreateDomain(friendlyName, configFile);
var objectHandle = sandbox.CreateInstance(typeof(T).Assembly.FullName, typeof(T).FullName, true, BindingFlags.Default, null, args, null, null, null);
T sandBoxAction = objectHandle.Unwrap() as T;
sandBoxAction.Run();
Tuple<AppDomain, SandboxAction> box = new Tuple<AppDomain, SandboxAction>(sandbox, sandBoxAction);
_sandboxes.Add(box);
return sandBoxAction;
}
private static AppDomain CreateDomain(string name, string customConfigFile)
{
FileInfo info = customConfigFile != null ? new FileInfo(customConfigFile) : null;
if (!string.IsNullOrEmpty(customConfigFile) && !info.Exists)
throw new ArgumentException("customConfigFile not found using " + customConfigFile + " at " + info.FullName);
var appsetup = new AppDomainSetup();
//appsetup.ApplicationBase = Path.GetDirectoryName(typeof(Sandbox).Assembly.Location);
appsetup.ApplicationBase = AppDomain.CurrentDomain.SetupInformation.ApplicationBase;
if (customConfigFile==null)
customConfigFile = AppDomain.CurrentDomain.SetupInformation.ConfigurationFile;
appsetup.ConfigurationFile = customConfigFile;
var sandbox = AppDomain.CreateDomain(
name,
AppDomain.CurrentDomain.Evidence,
appsetup);
return sandbox;
}
public static void DestroyAppDomainForSandbox(SandboxAction action)
{
foreach(var tuple in _sandboxes)
{
if(tuple.Second == action)
{
AppDomain.Unload(tuple.First);
Console.WriteLine("Unloaded sandbox ");
_sandboxes.Remove(tuple);
return;
}
}
}
}
[Serializable]
public abstract class SandboxAction : MarshalByRefObject
{
public override object InitializeLifetimeService()
{
return null;
}
public void Run()
{
string name = AppDomain.CurrentDomain.FriendlyName;
Log.Info("Executing {0} in AppDomain:{1} thread:{2}", name, AppDomain.CurrentDomain.Id, Thread.CurrentThread.ManagedThreadId);
try
{
OnRun();
}
catch (Exception ex)
{
Log.Error(ex, "Exception in app domain {0}", name);
throw;
}
}
protected abstract void OnRun();
public virtual void Stop()
{
}
}