I'm trying to run a job in the background using resque and after following the docs and a couple of tutorials I am stuck. Running the resque.log it says the job ran but it really didn't.
I am currently calling the job like
Resque.enqueue(CsvImporterJob, params[:file].path, #organization.id)
This is the job:
class CsvImporterJob < ActiveJob::Base
#queue = :import
def self.perform(file, organization_id)
CSV.foreach(file, :headers => true) do |row|
if row
user = row.to_hash
email = user["email"]
case user["role"].downcase
when "admin"
role_id = Role.admin
else
role_id = Role.user
end
ActivateHelper.send_activation_token(email, organization_id, role_id)
end
end
end
end
This is the log:
D, [2016-04-06T11:02:59.854604 #6548] DEBUG -- : resque-1.25.2: Waiting for *
D, [2016-04-06T11:03:04.856115 #6548] DEBUG -- : Checking import
D, [2016-04-06T11:03:04.856388 #6548] DEBUG -- : Found job on import
I, [2016-04-06T11:03:04.856441 #6548] INFO -- : got: (Job{import} | CsvImporterJob | ["/var/folders/92/87zkvh_53mb3721x4025vmsr0000gn/T/RackMultipart20160406-6422-1t7akw1.csv", 6])
D, [2016-04-06T11:03:04.856771 #6548] DEBUG -- : resque-1.25.2: Processing import since 1459954984 [CsvImporterJob]
I, [2016-04-06T11:03:04.856811 #6548] INFO -- : Running before_fork hooks with [(Job{import} | CsvImporterJob | ["/var/folders/92/87zkvh_53mb3721x4025vmsr0000gn/T/RackMultipart20160406-6422-1t7akw1.csv", 6])]
D, [2016-04-06T11:03:04.858903 #6548] DEBUG -- : resque-1.25.2: Forked 6608 at 1459954984
I, [2016-04-06T11:03:04.859985 #6608] INFO -- : Running after_fork hooks with [(Job{import} | CsvImporterJob | ["/var/folders/92/87zkvh_53mb3721x4025vmsr0000gn/T/RackMultipart20160406-6422-1t7akw1.csv", 6])]
I, [2016-04-06T11:03:04.865640 #6608] INFO -- : done: (Job{import} | CsvImporterJob | ["/var/folders/92/87zkvh_53mb3721x4025vmsr0000gn/T/RackMultipart20160406-6422-1t7akw1.csv", 6])
D, [2016-04-06T11:03:04.869546 #6548] DEBUG -- : Checking import
D, [2016-04-06T11:03:04.869914 #6548] DEBUG -- : Sleeping for 5.0 seconds
D, [2016-04-06T11:03:04.869947 #6548] DEBUG -- : resque-1.25.2: Waiting for *
D, [2016-04-06T11:03:09.871048 #6548] DEBUG -- : Checking import
D, [2016-04-06T11:03:09.871237 #6548] DEBUG -- : Sleeping for 5.0 seconds
D, [2016-04-06T11:03:09.871459 #6548] DEBUG -- : resque-1.25.2: Waiting for *
D, [2016-04-06T11:03:14.872187 #6548] DEBUG -- : Checking import
D, [2016-04-06T11:03:14.872390 #6548] DEBUG -- : Sleeping for 5.0 seconds
Anything specific I should be doing? Thanks in advance
Solved this problem by setting up the connection with redis correctly
require 'resque/tasks'
require 'resque/scheduler/tasks'
task "resque:setup" => :environment do
ENV['QUEUE'] = '*'
Resque.redis = 'localhost:6379' unless Rails.env == 'production'
Resque.before_fork = Proc.new do |job|
ActiveRecord::Base.connection.disconnect!
end
Resque.after_fork = Proc.new do |job|
ActiveRecord::Base.establish_connection
end
end
Related
I'm inconsistently getting this error in a first experiment with OCaml 5.0.0~beta1:
Fatal error: exception Stdlib.Effect.Unhandled(Domainslib__Task.Wait(_, _))
My setup:
Processor: Intel(R) Core(TM) i7-8750H CPU # 2.20GHz
Debian 10 (buster)
opam version 2.1.3 installed as binary from this script
opam switch: "→ 5.0.0~beta1 ocaml-base-compiler.5.0.0~beta1 5.0.0~beta1"
After a quick read of this tutorial, I copied the parallel_matrix_multiply function and added some code in the end just to use it:
open Domainslib
let parallel_matrix_multiply pool a b =
let i_n = Array.length a in
let j_n = Array.length b.(0) in
let k_n = Array.length b in
let res = Array.make_matrix i_n j_n 0 in
Task.parallel_for pool ~start:0 ~finish:(i_n - 1) ~body:(fun i ->
for j = 0 to j_n - 1 do
for k = 0 to k_n - 1 do
res.(i).(j) <- res.(i).(j) + a.(i).(k) * b.(k).(j)
done
done);
res ;;
let pool = Task.setup_pool ~num_domains:3 () in
let a = Array.make_matrix 2 2 1 in
let b = Array.make_matrix 2 2 2 in
let c = parallel_matrix_multiply pool a b in
for i = 0 to 1 do
for j = 0 to 1 do
Printf.printf "%d " c.(i).(j)
done;
print_char '\n'
done;;
I then compile it with no errors with
ocamlfind ocamlopt -linkpkg -package domainslib parallel_for.ml
and then comes the problem: executing the generated a.out file sometimes (rarely) prints the expected output
4 4
4 4
but usually ends with the error mentioned earlier:
Fatal error: exception Stdlib.Effect.Unhandled(Domainslib__Task.Wait(_, _))
Sorry if I am making some trivial mistake, but I can't understand what is going on, especially given that the error happens inconsistently.
The parallel_matrix_multiply computation is running outside of the Domainslib scheduler, thus whenever a task yields to the scheduler, the Wait effect is unhandled and transformed into a Effect.Unhandled exception.
The solution is to run the parallel computation within Task.run:
...
let c = Task.run pool (fun () -> parallel_matrix_multiply pool a b) in
...
i'm trying to run a test command in sbt-shell of intellij ide for this -> [https://github.com/theiterators/akka-http-microservice#akka-http-microservice-example] project then it shows Akka version conflict. how to resolve it?
this is build.sbt
enablePlugins(JavaAppPackaging , GatlingPlugin)
name := "akka-http-microservice"
organization := "com.theiterators"
version := "1.0"
scalaVersion := "2.13.5"
scalacOptions := Seq("-unchecked", "-deprecation", "-encoding", "utf8",
"-target:jvm-1.8",
"-feature",
"-language:implicitConversions",
"-language:postfixOps")
libraryDependencies ++= {
val akkaHttpV = "10.2.4"
val akkaV = "2.6.14"
val scalaTestV = "3.2.8"
val circeV = "0.13.0"
val akkaHttpCirceV = "1.36.0"
val gatlingVersion = "3.5.1"
Seq(
"com.typesafe.akka" %% "akka-actor" % akkaV,
"com.typesafe.akka" %% "akka-stream" % akkaV,
"com.typesafe.akka" %% "akka-http" % akkaHttpV,
"io.circe" %% "circe-core" % circeV,
"io.circe" %% "circe-generic" % circeV,
"de.heikoseeberger" %% "akka-http-circe" % akkaHttpCirceV,
"com.typesafe.akka" %% "akka-testkit" % akkaV,
"com.typesafe.akka" %% "akka-http-testkit" % akkaHttpV % "test",
"org.scalatest" %% "scalatest" % scalaTestV % "test",
"io.gatling.highcharts" % "gatling-charts-highcharts" % gatlingVersion % "test,it",
"io.gatling" % "gatling-test-framework" % gatlingVersion % "test,it"
)
}
Revolver.settings
this is plugin.sbt
addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.1")
addSbtPlugin("io.gatling" % "gatling-sbt" % "3.2.2")
addSbtPlugin("io.gatling" % "gatling-sbt" % "MANUALLY_REPLACE_WITH_LATEST_VERSION")
this is service specification
import akka.event.NoLogging
import akka.http.scaladsl.model.ContentTypes._
import akka.http.scaladsl.model.{HttpRequest, HttpResponse}
import akka.http.scaladsl.model.StatusCodes._
import akka.http.scaladsl.testkit.ScalatestRouteTest
import akka.stream.scaladsl.Flow
import org.scalatest.flatspec.AsyncFlatSpec
import org.scalatest.matchers.should.Matchers
class ServiceSpec extends AsyncFlatSpec with Matchers with ScalatestRouteTest with Service with Protocols {
override def testConfigSource = "akka.loglevel = WARNING"
override def config = testConfig
override val logger = NoLogging
val ip1Info = IpInfo("8.8.8.8", Option("United States"), Option("Mountain View"), Option(37.386), Option(-122.0838))
val ip2Info = IpInfo("8.8.4.4", Option("United States"), None, Option(38.0), Option(-97.0))
val ipPairSummary = IpPairSummary(ip1Info, ip2Info)
override lazy val ipApiConnectionFlow = Flow[HttpRequest].map { request =>
if (request.uri.toString().endsWith(ip1Info.query))
HttpResponse(status = OK, entity = marshal(ip1Info))
else if(request.uri.toString().endsWith(ip2Info.query))
HttpResponse(status = OK, entity = marshal(ip2Info))
else
HttpResponse(status = BadRequest, entity = marshal("Bad ip format"))
}
"Service" should "respond to single IP query" in {
Get(s"/ip/${ip1Info.query}") ~> routes ~> check {
status shouldBe OK
contentType shouldBe `application/json`
responseAs[IpInfo] shouldBe ip1Info
}
Get(s"/ip/${ip2Info.query}") ~> routes ~> check {
status shouldBe OK
contentType shouldBe `application/json`
responseAs[IpInfo] shouldBe ip2Info
}
}
it should "respond to IP pair query" in {
Post(s"/ip", IpPairSummaryRequest(ip1Info.query, ip2Info.query)) ~> routes ~> check {
status shouldBe OK
contentType shouldBe `application/json`
responseAs[IpPairSummary] shouldBe ipPairSummary
}
}
it should "respond with bad request on incorrect IP format" in {
Get("/ip/asdfg") ~> routes ~> check {
status shouldBe BadRequest
responseAs[String].length should be > 0
}
Post(s"/ip", IpPairSummaryRequest(ip1Info.query, "asdfg")) ~> routes ~> check {
status shouldBe BadRequest
responseAs[String].length should be > 0
}
Post(s"/ip", IpPairSummaryRequest("asdfg", ip1Info.query)) ~> routes ~> check {
status shouldBe BadRequest
responseAs[String].length should be > 0
}
}
}
Error given below--
[info] ServiceSpec *** ABORTED ***
[info] java.lang.IllegalStateException: You are using version 2.6.14 of Akka, but it appears you (perhaps indirectly) also depend on older versions of related artifacts. You can solve this by adding an explicit dependency on version 2.6.14 of the [akka-slf4j] artifacts to your project. Here's a complete collection of detected artifacts: (2.6.11, [akka-slf4j]), (2.6.14, [akka-actor, akka-protobuf-v3, akka-stream, akka-testkit]). See also: https://doc.akka.io/docs/akka/current/common/binary-compatibility-rules.html#mixed-versioning-is-not-allowed
[info] at akka.util.ManifestInfo.checkSameVersion(ManifestInfo.scala:184)
[info] at akka.util.ManifestInfo.checkSameVersion(ManifestInfo.scala:162)
[info] at akka.actor.ActorSystemImpl.liftedTree2$1(ActorSystem.scala:1033)
[info] at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:1022)
[info] at akka.actor.ActorSystemImpl._start(ActorSystem.scala:1022)
[info] at akka.actor.ActorSystemImpl.start(ActorSystem.scala:1045)
[info] at akka.actor.ActorSystem$.apply(ActorSystem.scala:272)
[info] at akka.actor.ActorSystem$.apply(ActorSystem.scala:316)
[info] at akka.actor.ActorSystem$.apply(ActorSystem.scala:290)
[info] at akka.http.scaladsl.testkit.RouteTest.createActorSystem(RouteTest.scala:36)
[info] ...
[info] Run completed in 1 second, 568 milliseconds.
[info] Total number of tests run: 0
[info] Suites: completed 0, aborted 1
[info] Tests: succeeded 0, failed 0, canceled 0, ignored 0, pending 0
[info] *** 1 SUITE ABORTED ***
[error] Error during tests:
[error] ServiceSpec
[error] (Test / test) sbt.TestsFailedException: Tests unsuccessful
[error] Total time: 8 s, completed 22 Apr. 2021, 3:14:59 pm
Adding
"com.typesafe.akka" %% "akka-slf4j" % akkaV
to the Seq in your libraryDependencies block should resolve this. In general in Akka it's best to not rely on transitive dependencies.
I am facing a performance issue in Akka remoting. I have 2 actors Actor1 and Actor2. The message sending between the actor is synchronous ask request from Actor1 to Actor2 and the response back from Actor2 to Actor1. Below is the sample code snippets and config of my Actor:
Actor1.java:
object Actor1 extends App {
val conf = ConfigFactory.load()
val system = ActorSystem("testSystem1", conf.getConfig("remote1"))
val actor = system.actorOf(Props[Actor1].withDispatcher("my-dispatcher"), "actor1")
implicit val timeOut: Timeout = Timeout(10 seconds)
class Actor1 extends Actor {
var value = 0
var actorRef: ActorRef = null
override def preStart(): Unit = {
println(self.path)
}
override def receive: Receive = {
case "register" =>
actorRef = sender()
println("Registering the actor")
val time = System.currentTimeMillis()
(1 to 300000).foreach(value => {
if (value % 10000 == 0) {
println("message count -- " + value + " --- time taken - " + (System.currentTimeMillis() - time))
}
Await.result(actorRef ? value, 10 seconds)
})
val totalTime = System.currentTimeMillis() - time
println("Total Time - " + totalTime)
}
}
}
Actor2.java:
object Actor2 extends App {
val conf = ConfigFactory.load()
val system = ActorSystem("testSystem1", conf.getConfig("remote2"))
val actor = system.actorOf(Props[Actor2].withDispatcher("my-dispatcher"), "actor2")
implicit val timeOut: Timeout = Timeout(10 seconds)
actor ! "send"
class Actor2 extends Actor {
var value = 0
var actorSelection: ActorSelection = context.actorSelection("akka://testSystem1#127.0.0.1:6061/user/actor1")
override def receive: Receive = {
case "send" =>
actorSelection ! "register"
case int: Int => {
sender() ! 1
}
}
}
}
application.conf:
remote1 {
my-dispatcher {
executor = "thread-pool-executor"
type = PinnedDispatcher
}
akka {
actor {
provider = remote
}
remote {
artery {
transport = tcp # See Selecting a transport below
canonical.hostname = "127.0.0.1"
canonical.port = 6061
}
}
}
}
remote2 {
my-dispatcher {
executor = "thread-pool-executor"
type = PinnedDispatcher
}
akka {
actor {
provider = remote
}
remote {
artery {
transport = tcp # See Selecting a transport below
canonical.hostname = "127.0.0.1"
canonical.port = 6062
}
}
}
}
Output:
message count -- 10000 --- time taken - 5871
message count -- 20000 --- time taken - 9043
message count -- 30000 --- time taken - 12198
message count -- 40000 --- time taken - 15363
message count -- 50000 --- time taken - 18649
message count -- 60000 --- time taken - 22074
message count -- 70000 --- time taken - 25487
message count -- 80000 --- time taken - 28820
message count -- 90000 --- time taken - 32118
message count -- 100000 --- time taken - 35634
message count -- 110000 --- time taken - 39146
message count -- 120000 --- time taken - 42539
message count -- 130000 --- time taken - 45997
message count -- 140000 --- time taken - 50013
message count -- 150000 --- time taken - 53466
message count -- 160000 --- time taken - 57117
message count -- 170000 --- time taken - 61246
message count -- 180000 --- time taken - 65051
message count -- 190000 --- time taken - 68809
message count -- 200000 --- time taken - 72908
message count -- 210000 --- time taken - 77091
message count -- 220000 --- time taken - 80855
message count -- 230000 --- time taken - 84679
message count -- 240000 --- time taken - 89089
message count -- 250000 --- time taken - 93132
message count -- 260000 --- time taken - 97360
message count -- 270000 --- time taken - 101442
message count -- 280000 --- time taken - 105656
message count -- 290000 --- time taken - 109665
message count -- 300000 --- time taken - 113706
Total Time - 113707
Is there any wrong I am doing here?. Any observation or suggestion to improve the performance?
The main issue I see with the code is Await.result(). That is a blocking operation, and will most likely affect performance.
I suggest collecting the results in a fixed array / list, use an integer as an array, and consider it complete when the expected number of responses have been received.
I'm trying to make build-system runs unit-test automatically.
The problem is that test output was duplicated when i use -j option.
have possible solution...?
Parallel build + consistent output
similar to
function()
synchronized {
add_custom_command("...run test...")
}
endfunction()
which is no cmake syntax
src - https://github.com/dearblanc/cmake_practice
< first build >
< second build > - make failure by intention
< wow >
Running module_a_test
Running module_b_test
[==========] Running 1 test from 1 test suite.
[----------] Global test environment set[-=u=p=.=
=[=-=-=-=-=-]- -R-u-n-n]i n1g t2e stte sftrso mf rMoOmD U2L Et_eAs
t[ sRuUiNt e s .
[ -]- -M-O-D-U-L-E-_-A]. GGElTo
bCa:l\ Utseesrts \ednevairrbo\nvmsecnotd es_ewto-rukps.p
a[c-e-\-c-m-a-k-e-_-p-r]a c1t itcee\smto dfurloem_ aM\OsDrUcL\Em_oBd2u
l[e _RaU_Nt e s t . c c]: 7M:O DFUaLiEl_uBr2e.G
EETx
p[e c t e d e qOuKa l]i tMyO DoUfL Et_hBe2s.eG EvTa l(u0e sm:s
)
["-A-A-"-
- - -a-.-g-e]t (1)
t e s t Wfhriocmh MiOsD:U L"EA_"B
2 ([8 mFsA ItLoEtDal)
][ -M-O-D-U-L-E-_-A-.-G]E T1 (t4e5s tm sf)r
o[m- -M-O-D-U-L-E-_-B-1]
[1 RtUeNs t f r o m] MMOODDUULLEE__AB 1(.1G0E1T
mCs: \tUostearls)\
d
e[a-r-b-\-v-s-c-o-d-e-_]w oGrlkosbpaalc et\ecsmta keen_vpirraocntmiecnet\ mtoedaurl-ed_obw\ns
r[c=\=m=o=d=u=l=e=_=b=1]_ t1e stte.sctc :f6r:o mF a1i ltuerset
Esxupietcet erda ne.q u(a1l5i5t ym so ft otthaels)e
[v a lPuAeSsS:E
D "]B B01 "t
e s tbs1..
g[e t (F)A
I L E D W h]i c1h tiess:t ," Bl1"i
s[t e dF AbIeLlEoDw :
][ M OFDAUILLEE_DB 1 .]G ETM O(D3U9L Em_sA).
[G-E-T-
-
- -1- -F-A-I]L E1D tTeEsStT
from MODULE_B1 (96 ms total)
I am getting an error running the following Erlang code on an Ubuntu Server machine, it runs properly on my machine running Solus. I am new to Erlang and am not sure how to read the error, as all other examples only have one function and module in the error code.
I have two files
Sensor:
-module(sensor).
-export([start/2, senseAndReport/3]).
start(WatcherPid, SensorId) ->
Ref = make_ref(),
senseAndReport(WatcherPid, SensorId, Ref).
senseAndReport(WatcherPid, SensorId, Ref) ->
Measurement = rand:uniform(11),
if
Measurement == 11 ->
WatcherPid ! {kill, {self(), SensorId}, error},
exit(error);
true ->
WatcherPid ! {Measurement, {self(), SensorId}, Ref}
end,
receive
{ok, Ref} ->
Sleep_time = rand:uniform(10000),
timer:sleep(Sleep_time),
senseAndReport(WatcherPid, SensorId, Ref)
end.
And Watcher:
-module(watcher).
-export([start/1, createSensor/3, restartASensor/2, watch/1]).
start(NumOfSensor) ->
if
NumOfSensor == 0 ->
io:format("Please enter a number greater than 0.~n");
true ->
createSensor(NumOfSensor, [], 0)
end.
createSensor(NumOfSensor, SensorList, SensorId) ->
if
length(SensorList) == 10 ->
io:format("Start watching:~n"),
[io:format(" Id: ~p, Pid: ~p~n", [Id, Pid]) || {Id, Pid} <- SensorList],
if NumOfSensor /= 0 ->
spawn(watcher, createSensor, [NumOfSensor, [], SensorId]),
watch(SensorList);
true ->
watch(SensorList)
end;
NumOfSensor == 0 ->
io:format("Start watching~n"),
[io:format(" Id: ~p, Pid: ~p~n", [Id, Pid]) || {Id, Pid} <- SensorList],
watch(SensorList);
true ->
SensorPid = spawn_monitor(sensor, start, [self(), SensorId]),
createSensor(NumOfSensor - 1, lists:merge(SensorList, [{SensorId, SensorPid}]), SensorId + 1)
end.
restartASensor(SensorList, SensorId) ->
{SensorPid, _} = spawn_monitor(sensor, start, [self(), SensorId]),
io:format(" Restarted sensor: ~p, new Pid: ~p~n", [SensorId, SensorPid]),
NewList = lists:merge(SensorList, [{SensorId, SensorPid}]),
watch(NewList).
watch(SensorList) ->
receive
{kill, {From, FromId}, error} ->
io:format(" Sensor ~p died~n", [FromId]),
restartASensor(lists:delete({FromId, From}, SensorList), FromId);
{Measurement, {From, FromId}, Ref} ->
io:format("MSG: ~2p, From sensor ~4p~n", [Measurement, FromId]),
From ! {ok, Ref},
watch(SensorList)
end.
This give me the following output:
Eshell V5.8.5 (abort with ^G)
1> c(watcher).
{ok,watcher}
2> watcher:start(1).
Start watching
Id: 0, Pid: <0.39.0>
=ERROR REPORT==== 18-Nov-2016::19:32:35 ===
Error in process <0.39.0> with exit value: {undef,[{rand,uniform,[11]},{sensor,senseAndReport,3}]}
Dogbert's comment is the correct answer. The version of Erlang I was using was older on the Ubuntu machine, needed to replace the rand module with the random module.