Running Apache Beam DataFlow, I was able to run a Streaming pipeline successfully for 12 days (November 5-17), then DataFlow job stopped processing data.
I see SSL errors when contacting AI Platform prediction and DataFlow shows:
Processing stuck in step <step_id> for at least <time_interval> without outputting or completing in state finish at <stack_trace>
Is it enough by handling SSL exception?, what is the best way to prevent this deadlock in DataFlow.
Related article here
Version
Streaming Job
Python 2.7 Apache Beam GCP: 2.16.0
Flow
Tweet Python listener (Reads Tweets using tweepy) -> PubSub -> DataFlow (Reads data from PubSub, calls AI Platform prediction) -> BigQuery
Listener code
DataFlow code
I tried to access my data today and noticed that Pipeline stopped processing back in November 17th. I see this error:
2019-12-06 21:15:26.960 PSTError message from worker: Processing stuck in step s02 for at least 476h25m00s without outputting or completing in state finish at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1693) at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323) at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1729) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895) at org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57) at org.apache.beam.runners.dataflow.worker.fn.control.RegisterAndProcessBundleOperation.finish(RegisterAndProcessBundleOperation.java:330) at org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:85) at org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125) at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.process(StreamingDataflowWorker.java:1320) at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.access$1000(StreamingDataflowWorker.java:151) at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker$6.run(StreamingDataflowWorker.java:1053) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
I also see this error:
java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error received from SDK harness for instruction -2222113: Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 158, in _execute
response = task()
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 191, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 343, in do_instruction
request.instruction_id)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 369, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 663, in process_bundle
data.ptransform_id].process_encoded(data.data)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 143, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/worker/operations.py", line 255, in apache_beam.runners.worker.operations.Operation.output
def output(self, windowed_value, output_index=0):
File "apache_beam/runners/worker/operations.py", line 256, in apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 834, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 587, in apache_beam.runners.common.PerWindowInvoker.invoke_process
self._invoke_process_per_window(
File "apache_beam/runners/common.py", line 659, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 880, in apache_beam.runners.common._OutputProcessor.process_outputs
def process_outputs(self, windowed_input_element, results):
File "apache_beam/runners/common.py", line 919, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 834, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 587, in apache_beam.runners.common.PerWindowInvoker.invoke_process
self._invoke_process_per_window(
File "apache_beam/runners/common.py", line 659, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 880, in apache_beam.runners.common._OutputProcessor.process_outputs
def process_outputs(self, windowed_input_element, results):
File "apache_beam/runners/common.py", line 919, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 849, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise_with_traceback(new_exn)
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 441, in apache_beam.runners.common.SimpleInvoker.invoke_process
windowed_value, self.process_method(windowed_value.value))
File "streaming_twitter.py", line 203, in <lambda>
File "streaming_twitter.py", line 112, in estimate
File "streaming_twitter.py", line 96, in prediction
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 851, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 165, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/google_auth_httplib2.py", line 198, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 2133, in request
cachekey,
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1796, in _request
conn, request_uri, method, body, headers
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1737, in _conn_request
response = conn.getresponse()
File "/usr/lib/python2.7/httplib.py", line 1121, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "/usr/lib/python2.7/socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "/usr/lib/python2.7/ssl.py", line 766, in recv
return self.read(buflen)
File "/usr/lib/python2.7/ssl.py", line 653, in read
v = self._sslobj.read(len)
RuntimeError: error: [Errno 0] Error [while running 'generatedPtransform-2222099']
java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57)
org.apache.beam.runners.dataflow.worker.fn.control.RegisterAndProcessBundleOperation.finish(RegisterAndProcessBundleOperation.java:330)
org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:85)
org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125)
org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.process(StreamingDataflowWorker.java:1320)
org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.access$1000(StreamingDataflowWorker.java:151)
org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker$6.run(StreamingDataflowWorker.java:1053)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: Error received from SDK harness for instruction -2222113: Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 158, in _execute
response = task()
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 191, in <lambda>
self._execute(lambda: worker.do_instruction(work), work)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 343, in do_instruction
request.instruction_id)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 369, in process_bundle
bundle_processor.process_bundle(instruction_id))
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 663, in process_bundle
data.ptransform_id].process_encoded(data.data)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 143, in process_encoded
self.output(decoded_value)
File "apache_beam/runners/worker/operations.py", line 255, in apache_beam.runners.worker.operations.Operation.output
def output(self, windowed_value, output_index=0):
File "apache_beam/runners/worker/operations.py", line 256, in apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 834, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 587, in apache_beam.runners.common.PerWindowInvoker.invoke_process
self._invoke_process_per_window(
File "apache_beam/runners/common.py", line 659, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 880, in apache_beam.runners.common._OutputProcessor.process_outputs
def process_outputs(self, windowed_input_element, results):
File "apache_beam/runners/common.py", line 919, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 834, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 587, in apache_beam.runners.common.PerWindowInvoker.invoke_process
self._invoke_process_per_window(
File "apache_beam/runners/common.py", line 659, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 880, in apache_beam.runners.common._OutputProcessor.process_outputs
def process_outputs(self, windowed_input_element, results):
File "apache_beam/runners/common.py", line 919, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 143, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive
self.consumer.process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 593, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 594, in apache_beam.runners.worker.operations.DoOperation.process
delayed_application = self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 776, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 782, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 849, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise_with_traceback(new_exn)
File "apache_beam/runners/common.py", line 780, in apache_beam.runners.common.DoFnRunner.process
return self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 441, in apache_beam.runners.common.SimpleInvoker.invoke_process
windowed_value, self.process_method(windowed_value.value))
File "streaming_twitter.py", line 203, in <lambda>
File "streaming_twitter.py", line 112, in estimate
File "streaming_twitter.py", line 96, in prediction
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper
return wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 851, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 165, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/google_auth_httplib2.py", line 198, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 2133, in request
cachekey,
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1796, in _request
conn, request_uri, method, body, headers
File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1737, in _conn_request
response = conn.getresponse()
File "/usr/lib/python2.7/httplib.py", line 1121, in getresponse
response.begin()
File "/usr/lib/python2.7/httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "/usr/lib/python2.7/httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "/usr/lib/python2.7/socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "/usr/lib/python2.7/ssl.py", line 766, in recv
return self.read(buflen)
File "/usr/lib/python2.7/ssl.py", line 653, in read
v = self._sslobj.read(len)
RuntimeError: error: [Errno 0] Error [while running 'generatedPtransform-2222099']
org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:157)
org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:140)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:249)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.Contexts$ContextualizedServerCallListener.onMessage(Contexts.java:76)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:297)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:738)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
org.apache.beam.vendor.grpc.v1p21p0.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
Processing stuck in step s02 for at least 05h20m00s without outputting or completing in state process
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.beam.runners.dataflow.worker.fn.data.RemoteGrpcPortWriteOperation.maybeWait(RemoteGrpcPortWriteOperation.java:175)
at org.apache.beam.runners.dataflow.worker.fn.data.RemoteGrpcPortWriteOperation.process(RemoteGrpcPortWriteOperation.java:196)
at org.apache.beam.runners.dataflow.worker.util.common.worker.OutputReceiver.process(OutputReceiver.java:49)
at org.apache.beam.runners.dataflow.worker.util.common.worker.ReadOperation.runReadLoop(ReadOperation.java:201)
at org.apache.beam.runners.dataflow.worker.util.common.worker.ReadOperation.start(ReadOperation.java:159)
at org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:77)
at org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125)
at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.process(StreamingDataflowWorker.java:1320)
at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.access$1000(StreamingDataflowWorker.java:151)
at org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker$6.run(StreamingDataflowWorker.java:1053)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Are you running some extremely slow operation during bundle finalization (DoFn.finish_bundle or DoFn.teardown) (for example, a per-element RPC call) ? If so please try to optimize to do that in batches. As the page you referred to points out, "Processing stuck" warning just means that DoFn did not transition out of a given state for a long period. Could be due to something being very slow or actually being stuck, for example while performing some RPC. If actually stuck, please try introducing a timeout to that operation.
Related
I had issues in accessing S3 buckets from EC2. I deleted both ~/.aws/config and ~/.aws/credentials files. Now, when I try to create the profiles again, I get this long list of exceptions:
$ aws configure --profile dev
Traceback (most recent call last): File "/usr/local/bin/aws", line
27, in
sys.exit(main()) File "/usr/local/bin/aws", line 23, in main
return awscli.clidriver.main() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 50, in main
return driver.main() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 176, in
main
parser = self._create_parser() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 157, in
_create_parser
command_table = self._get_command_table() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 91, in
_get_command_table
self._command_table = self._build_command_table() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 111, in
_build_command_table
command_object=self) File "/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
672, in emit
return self._events.emit(event_name, **kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 356,
in emit
return self._emitter.emit(aliased_event_name, **kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 228,
in emit
return self._emit(event_name, kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 211,
in _emit
response = handler(**kwargs) File "/usr/lib/python3/dist-packages/awscli/customizations/preview.py",
line 70, in mark_as_preview
service_name=original_command.service_model.service_name, File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 351, in
service_model
return self._get_service_model() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 368, in
_get_service_model
api_version = self.session.get_config_variable('api_versions').get( File
"/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
234, in get_config_variable
logical_name) File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 227, in get_config_variable
return provider.provide() File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 324, in provide
value = provider.provide() File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 383, in provide
config = self._session.get_scoped_config() File "/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
335, in get_scoped_config
raise ProfileNotFound(profile=profile_name) botocore.exceptions.ProfileNotFound: The config profile (dev) could
not be found [~] aws configure --profile dev Traceback (most recent
call last): File "/usr/local/bin/aws", line 27, in
sys.exit(main()) File "/usr/local/bin/aws", line 23, in main
return awscli.clidriver.main() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 50, in main
return driver.main() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 176, in
main
parser = self._create_parser() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 157, in
_create_parser
command_table = self._get_command_table() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 91, in
_get_command_table
self._command_table = self._build_command_table() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 111, in
_build_command_table
command_object=self) File "/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
672, in emit
return self._events.emit(event_name, **kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 356,
in emit
return self._emitter.emit(aliased_event_name, **kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 228,
in emit
return self._emit(event_name, kwargs) File "/usr/local/lib/python3.5/dist-packages/botocore/hooks.py", line 211,
in _emit
response = handler(**kwargs) File "/usr/lib/python3/dist-packages/awscli/customizations/preview.py",
line 70, in mark_as_preview
service_name=original_command.service_model.service_name, File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 351, in
service_model
return self._get_service_model() File "/usr/lib/python3/dist-packages/awscli/clidriver.py", line 368, in
_get_service_model
api_version = self.session.get_config_variable('api_versions').get( File
"/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
234, in get_config_variable
logical_name) File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 227, in get_config_variable
return provider.provide() File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 324, in provide
value = provider.provide() File "/usr/local/lib/python3.5/dist-packages/botocore/configprovider.py",
line 383, in provide
config = self._session.get_scoped_config() File "/usr/local/lib/python3.5/dist-packages/botocore/session.py", line
335, in get_scoped_config
raise ProfileNotFound(profile=profile_name) botocore.exceptions.ProfileNotFound: The config profile (dev) could
not be found
Its failing because you deleted those directories and like the error mentions, it can't find the profile.
You have to run:
$ aws configure
I manually updated ~/.aws/config file. This resolved the issue.
I run this code in my local pc it works, but when I tried to run it in google dataflow it gives me an error. I found the location of the error, which is in ParDo function. The first loop returns the value as desired however the second loop giving me an error. Can anybody suggest an alternative approach or figure out where is the issue.
The element['content'] is URL giving the location of the file with JSON data
class DimTrans(beam.DoFn):
def process(self, element):
import pandas as pd
import apache_beam as beam
data = pd.read_json(element['content'])
title = data[:1]
data = data[1:]
for idx, item in data.iterrows():
for dim, axis in item.items():
label = (title[dim].values[0])
d = {'axisDim': int(dim),
'axisOrder': int(idx),
'axisValue': float(axis),
'axisTitle': u'{}'.format(label)}
yield d
Error trace:
An exception was raised when trying to execute the workitem 533970500833477506 : Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line 642, in do_work work_executor.execute() File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py", line 156, in execute op.start() File "dataflow_worker/native_operations.py", line 38, in dataflow_worker.native_operations.NativeReadOperation.start def start(self): File "dataflow_worker/native_operations.py", line 39, in dataflow_worker.native_operations.NativeReadOperation.start with self.scoped_start_state: File "dataflow_worker/native_operations.py", line 44, in dataflow_worker.native_operations.NativeReadOperation.start with self.spec.source.reader() as reader: File "dataflow_worker/native_operations.py", line 54, in dataflow_worker.native_operations.NativeReadOperation.start self.output(windowed_value) File "apache_beam/runners/worker/operations.py", line 175, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 85, in apache_beam.runners.worker.operations.ConsumerSet.receive cython.cast(Operation, consumer).process(windowed_value) File "apache_beam/runners/worker/operations.py", line 403, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 404, in apache_beam.runners.worker.operations.DoOperation.process self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 569, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 577, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 602, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 575, in apache_beam.runners.common.DoFnRunner.process self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 352, in apache_beam.runners.common.SimpleInvoker.invoke_process output_processor.process_outputs( File "apache_beam/runners/common.py", line 673, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 85, in apache_beam.runners.worker.operations.ConsumerSet.receive cython.cast(Operation, consumer).process(windowed_value) File "apache_beam/runners/worker/operations.py", line 403, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 404, in apache_beam.runners.worker.operations.DoOperation.process self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 569, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 577, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 618, in apache_beam.runners.common.DoFnRunner._reraise_augmented six.reraise(type(new_exn), new_exn, original_traceback) File "apache_beam/runners/common.py", line 575, in apache_beam.runners.common.DoFnRunner.process self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 352, in apache_beam.runners.common.SimpleInvoker.invoke_process output_processor.process_outputs( File "apache_beam/runners/common.py", line 651, in apache_beam.runners.common._OutputProcessor.process_outputs for result in results: File "C:/Users/PycharmProjects/GCP/test1file.py", line 56, in process File "/usr/local/lib/python2.7/dist-packages/pandas/core/generic.py", line 2668, in getattr return object.getattribute(self, name) AttributeError: 'Series' object has no attribute 'items' [while running 'Print Results/ParDo(DimTrans)']
I'm running the following successful Apache Beam test pipeline on Google Dataflow. It uses Datastore as a source and sink. Many entities in our database are assigned to namespaces. This pipeline is meant to perform _do_work() on all entities of a certain kind in the given namespaces. Note, a similar test pipeline that does the same thing to non-namespaced entities also runs successfully:
import apache_beam as beam
from apache_beam.io.gcp.datastore.v1 import helper as apache_helper
from apache_beam.io.gcp.datastore.v1 import datastoreio
from google.cloud.proto.datastore.v1 import query_pb2
from googledatastore import helper
from .pipelines.dataflow_settings import (
PROJECT, NAMESPACES_PER_PIPELINE
)
class NamespacedDatastoreMigration(_DatastoreMigrationBase):
"""
Map a do-function over a query multiplexed across several namespaces.
The inheritor must implement the following:
- a PROJECT class attribute
- a do-function (_do_work())
- a method to get the namespaces across which to shard the query (
get_namespaces())
"""
_NAMESPACES_PER_PIPELINE = NAMESPACES_PER_PIPELINE # 25
def __init__(self, argv, migration_history_obj, model):
super(NamespacedDatastoreMigration, self).__init__(argv, migration_history_obj, model)
self._namespaces = self.get_namespaces()
self._pipelines = self._create_pipelines(argv)
def get_namespaces(self):
query_pb = query_pb2.Query()
helper.set_kind(query_pb, "__namespace__")
client = apache_helper.get_datastore(PROJECT)
namespace_entities = apache_helper.fetch_entities(PROJECT, '', query_pb, client)
namespaces = []
for n in namespace_entities:
# Get namespace name or id
key_path = n.key.path[-1]
if key_path.HasField('id'):
name_or_id = key_path.id
else:
name_or_id = key_path.name
# Avoid duplicates and test namespaces
if len(str(name_or_id)) > 1 and name_or_id not in namespaces:
namespaces.append(name_or_id)
return namespaces
def run(self):
for pipeline in self._pipelines:
pipeline.run()
def _create_pipelines(self, argv):
pipelines = []
for namespaces in zip(*[iter(self._namespaces)] * self._NAMESPACES_PER_PIPELINE):
p = beam.Pipeline(argv=argv)
(
(
p | 'ReadNamespace_{}'.format(
ns
) >> datastoreio.ReadFromDatastore(
project=self.PROJECT,
query=self.query(),
namespace=ns
)
for ns in namespaces
)
| 'JoinNamespaceEntities' >> beam.Flatten()
| self.__class__.__name__ >> beam.FlatMap(self._do_work)
| self._get_sink()
)
pipelines.append(p)
return pipelines
model = "App"
NamespacedDatastoreMigration(
argv,
kwargs.get('migration_history_obj'), # Irrelevant here
model # Entity kind
).run()
where argv is:
argv = [
'--project={0}'.format(PROJECT),
'--job_name=' + name, # A human readable descriptor that's been cleaned
'--staging_location=gs://{0}/migrations/'.format(BUCKET),
'--temp_location=gs://{0}/migrations/'.format(BUCKET),
'--setup_file=./setup.py',
'--runner=DataflowRunner'
]
This is based on the subclass:
class _DatastoreMigrationBase(object):
PROJECT = PROJECT
def __init__(self, argv, migration_history_obj, model):
self.migration_history_obj = migration_history_obj
if not model:
raise Exception('This operation requires a model class name.')
self.model = model
def query(self):
# Instantiate a filter protobuf
filter_pb = query_pb2.Filter()
# Get all non-deleted model instances
helper.set_property_filter(
filter_pb,
'deleted',
query_pb2.PropertyFilter.EQUAL,
False
)
# Instantiate a query protobuf
query_pb = query_pb2.Query(
filter=filter_pb
)
helper.set_kind(query_pb, self.model)
return query_pb
def _get_source(self):
return 'DatastoreRead' >> datastoreio.ReadFromDatastore(
self.PROJECT,
self.query(),
)
#staticmethod
def _do_work(entity):
return entity
def _get_sink(self):
return 'WriteToDatastore' >> datastoreio.WriteToDatastore(
self.PROJECT
)
However, when I subclass NamespacedDatastoreMigration like so:
from ..helpers import create_argv
from ..mappers import NamespacedDatastoreMigration
class CampaignActionField(NamespacedDatastoreMigration):
#staticmethod
def _do_work(entity):
target_url = entity.properties.get('target_url').string_value
message = entity.properties.get('message').string_value
path = entity.properties.get('path').string_value
if target_url and not message and not path:
entity.properties.get('action').string_value = 'webhook'
return entity
model = "Campaign" # Entity kind
CampaignActionField(
create_argv(kwargs.get('name')), # "ED-2275 Campaign action field"
kwargs.get('migration_history_obj'), # Irrelevant here
model
).run()
and this new pipeline runs on Dataflow, it fails. At first, it starts out okay. By that I mean I see the following INFO logs:
2018-11-20 (11:02:57) Worker configuration: n1-standard-1 in us-central1-b.
2018-11-20 (11:03:15) Starting 1 workers in us-central1-b.
# SEVERAL OF THE FOLLOWING FOR DIFFERENT NAMESPACES:
2018-11-20 (11:03:15) Executing operation ReadNamespace_xxxx_1/GroupByKey/Create
2018-11-20 (11:03:17) Executing operation ReadNamespace_xxxx_1/UserQuery/Read+ReadNamespace_xxxx_1/SplitQuery+ReadNa...
2018-11-20 (11:05:58) Executing operation ReadNamespace_xxxx_1/GroupByKey/Close
And then I get this traceback:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line 642, in do_work
work_executor.execute()
File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py", line 156, in execute
op.start()
File "dataflow_worker/shuffle_operations.py", line 49, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
def start(self):
File "dataflow_worker/shuffle_operations.py", line 50, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.scoped_start_state:
File "dataflow_worker/shuffle_operations.py", line 65, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.scoped_process_state:
File "dataflow_worker/shuffle_operations.py", line 66, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
with self.shuffle_source.reader() as reader:
File "dataflow_worker/shuffle_operations.py", line 70, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
self.output(windowed_value)
File "apache_beam/runners/worker/operations.py", line 168, in apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 88, in apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "dataflow_worker/shuffle_operations.py", line 229, in dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
with self.scoped_process_state:
File "dataflow_worker/shuffle_operations.py", line 236, in dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
self.output(wvalue.with_value((k, wvalue.value)))
File "apache_beam/runners/worker/operations.py", line 168, in apache_beam.runners.worker.operations.Operation.output
cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 88, in apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 423, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 424, in apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 673, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 679, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 702, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 677, in apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 413, in apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 787, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 88, in apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 423, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 424, in apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 673, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 679, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 702, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 677, in apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 413, in apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 787, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 88, in apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 423, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 424, in apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 673, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 679, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 702, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise
File "apache_beam/runners/common.py", line 677, in apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 413, in apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 787, in apache_beam.runners.common._OutputProcessor.process_outputs
self.main_receivers.receive(windowed_value)
File "apache_beam/runners/worker/operations.py", line 88, in apache_beam.runners.worker.operations.ConsumerSet.receive
cython.cast(Operation, consumer).process(windowed_value)
File "apache_beam/runners/worker/operations.py", line 423, in apache_beam.runners.worker.operations.DoOperation.process
with self.scoped_process_state:
File "apache_beam/runners/worker/operations.py", line 424, in apache_beam.runners.worker.operations.DoOperation.process
self.dofn_receiver.receive(o)
File "apache_beam/runners/common.py", line 673, in apache_beam.runners.common.DoFnRunner.receive
self.process(windowed_value)
File "apache_beam/runners/common.py", line 679, in apache_beam.runners.common.DoFnRunner.process
self._reraise_augmented(exn)
File "apache_beam/runners/common.py", line 717, in apache_beam.runners.common.DoFnRunner._reraise_augmented
raise_with_traceback(new_exn)
File "apache_beam/runners/common.py", line 677, in apache_beam.runners.common.DoFnRunner.process
self.do_fn_invoker.invoke_process(windowed_value)
File "apache_beam/runners/common.py", line 413, in apache_beam.runners.common.SimpleInvoker.invoke_process
output_processor.process_outputs(
File "apache_beam/runners/common.py", line 763, in apache_beam.runners.common._OutputProcessor.process_outputs
for result in results:
TypeError: 'Entity' object is not iterable [while running 's152-c260']
I assume this has something to do with the difference between the two _do_work() functions in NamespacedDatastoreMigration and CampaignActionField, since the former is successful while the latter fails, and _do_work() is the only difference between them (besides the entity kind being transformed). But I can't think of what exactly is going wrong and how to get around it. Does anyone have any thoughts?
Turns out that changing FlatMap to Map in NamespacedDatastoreMigration's _create_pipelines method fixed this for me. I also was stupidly calling NamespacedDatastoreMigration with a non-namespaced model, which is why it was succeeding while CamapaignActionField (which uses a namespaced model) was not.
I'm trying to use Flask-APNS, previously I check that my ck.pem (cert and key on one file) works on a php server.
But everytime I try to execute:
from apns import APNS
#apns = APNS(app, cert_file='ZivingCert.pem', key_file='ZivingKey.pem',passphrase='mypassphrase')
apns = APNS(app, cert_file='ck.pem', passphrase='mypassphrase')
apns.send_message(tokens=['fad71c0b27416f055bfb1617c8db4e55d1b98b412443e68fba65cfe59748b81a'], alert='hi world', extra={})
I get this error:
WARNING:apnsclient.backends.stdio:Failed to establish socket/SSL connection to ('gateway.sandbox.push.apple.com', 2195)
No handlers could be found for logger "apnsclient.backends.stdio"
Traceback (most recent call last):
File "/usr/local/Cellar/python/2.7.9/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 85, in run
self.result = application(self.environ, self.start_response)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__
return self.wsgi_app(environ, start_response)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1820, in wsgi_app
response = self.make_response(self.handle_exception(e))
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 263, in error_router
return original_handler(e)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1403, in handle_exception
reraise(exc_type, exc_value, tb)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 260, in error_router
return self.handle_error(e)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
response = self.full_dispatch_request()
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 263, in error_router
return original_handler(e)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 260, in error_router
return self.handle_error(e)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
rv = self.dispatch_request()
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 431, in wrapper
resp = resource(*args, **kwargs)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_httpauth.py", line 60, in decorated
return f(*args, **kwargs)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask/views.py", line 84, in view
return self.dispatch_request(*args, **kwargs)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/flask_restful/__init__.py", line 521, in dispatch_request
resp = meth(*args, **kwargs)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/scheduler/ws/v1/schedulesapns.py", line 21, in get
apns.send_message(tokens=['fad71c0b27416f055bfb1617c8db4e55d1b98b412443e68fba65cfe59748b81a'], alert='hi world', extra={})
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/scheduler/apns/__init__.py", line 134, in send_message
res = srv.send(message)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/apns.py", line 90, in send
status = self._connection.send(message)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/transport.py", line 255, in send
with self:
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/transport.py", line 228, in __enter__
self._open_connection() # can raise exception, bubblit up to the top
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/transport.py", line 519, in _open_connection
timeout=self.session.connect_timeout
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/backends/__init__.py", line 88, in get_cached_connection
return self.get_new_connection(address, certificate, timeout=timeout)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/backends/stdio.py", line 378, in get_new_connection
return self.connection_class(address, certificate, timeout=timeout)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/backends/stdio.py", line 142, in __init__
self._open_connection(timeout)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/backends/stdio.py", line 153, in _open_connection
self._connect_and_handshake()
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/apnsclient/backends/stdio.py", line 188, in _connect_and_handshake
self._connection.do_handshake()
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/OpenSSL/SSL.py", line 1442, in do_handshake
self._raise_ssl_error(self._ssl, result)
File "/Users/Ricardo/Documents/python/openshift/scheduler_env/lib/python2.7/site-packages/OpenSSL/SSL.py", line 1180, in _raise_ssl_error
raise SysCallError(-1, "Unexpected EOF")
SysCallError: (-1, 'Unexpected EOF')
I finally fixed thanks to logging messages.
To enable logging:
import logging
logging.basicConfig()
To fix WARNING:apnsclient.backends.stdio:Failed to establish socket/SSL connection to ('gateway.sandbox.push.apple.com', 2195) caused by security issue with SSL version 3.0 the Apple Push Notification server remove support for SSL 3.0 since Wednesday, October 29 of 2014.
def my_callback(key, reason):
print "my_callbaaaaaaack"
print str(key)
print str(reason)
import OpenSSL
OpenSSL.SSL.SSLv3_METHOD = OpenSSL.SSL.TLSv1_METHOD
from apns import APNS
apns = APNS(app, cert_file='ck.pem', passphrase='mypassphrase', failure_callback=my_callback)
I'm using haystack and solr on Django. I get UnicodeDecodeError and I think the reason is the unicode data in the database are either converted to str or decoded to utf-8 which makes a later UnicodeDecodeError when sending via httplib or printing to the console.
I think the problem should before the haystack full_prepare function, since as I checked afterwards data is already ruined.
Any thoughts?
I setup.pyed install (haystack, pysolr, django)
and used the binary of lxml from http://www.lfd.uci.edu/~gohlke/pythonlibs/
Microsoft Windows [Version 6.1.7601]
Copyright (c) 2009 Microsoft Corporation. All rights reserved.
C:\Users\Pouria\Desktop\conference\conference>python manage.py rebuild_index
WARNING: This will irreparably remove EVERYTHING from your search index in connection 'default'.
Your choices after this are to restore from backups or rebuild via the `rebuild_index` command.
Are you sure you wish to continue? [y/N] y
Removing all documents from your index because you said so.
Failed to clear Solr index: [Reason: Error 404 Not Found]
All documents removed.
C:\Python27\lib\site-packages\django\db\models\fields\__init__.py:808: RuntimeWarning: DateTimeField
received a naive datetime (2013-02-09 08:47:27.110000) while time zone support is active.
RuntimeWarning)
Indexing 2 conferences.
ERROR:root:Error updating conferences using default
Traceback (most recent call last):
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 210, in handle_label
self.update_backend(label, using)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 256, in update_backend
do_update(backend, index, qs, start, end, total, self.verbosity)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 78, in do_update
backend.update(index, current_qs)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\backends\solr_backend.py", line 72, i
n update
self.conn.add(docs, commit=commit, boost=index.get_field_weights())
File "C:\Python27\lib\site-packages\pysolr.py", line 786, in add
return self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher)
File "C:\Python27\lib\site-packages\pysolr.py", line 379, in _update
return self._send_request('post', path, message, {'Content-type': 'text/xml; charset=utf-8'})
File "C:\Python27\lib\site-packages\pysolr.py", line 291, in _send_request
timeout=self.timeout)
File "C:\Python27\lib\site-packages\requests\api.py", line 87, in post
return request('post', url, data=data, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies
)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 373, in send
r = adapter.send(request, **kwargs)
File "C:\Python27\lib\site-packages\requests\adapters.py", line 171, in send
timeout=timeout
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 421, in url
open
body=body, headers=headers)
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 273, in _ma
ke_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python27\lib\httplib.py", line 949, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 990, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 943, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 810, in _send_output
self.send(message_body)
File "C:\Python27\lib\httplib.py", line 775, in send
self.sock.sendall(str)
File "C:\Python27\lib\socket.py", line 222, in meth
return getattr(self._sock,name)(*args)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 143-149: ordinal not in range(
128)
Traceback (most recent call last):
File "manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 443, in execute_from
_command_line
utility.execute()
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 382, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 196, in run_from_argv
self.execute(*args, **options.__dict__)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 232, in execute
output = self.handle(*args, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\rebuild_index.py"
, line 15, in handle
call_command('update_index', **options)
File "C:\Python27\lib\site-packages\django\core\management\__init__.py", line 150, in call_command
return klass.execute(*args, **defaults)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 232, in execute
output = self.handle(*args, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 184, in handle
return super(Command, self).handle(*items, **options)
File "C:\Python27\lib\site-packages\django\core\management\base.py", line 341, in handle
label_output = self.handle_label(label, **options)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 210, in handle_label
self.update_backend(label, using)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 256, in update_backend
do_update(backend, index, qs, start, end, total, self.verbosity)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\management\commands\update_index.py",
line 78, in do_update
backend.update(index, current_qs)
File "C:\Users\Pouria\Desktop\conference\conference\haystack\backends\solr_backend.py", line 72, i
n update
self.conn.add(docs, commit=commit, boost=index.get_field_weights())
File "C:\Python27\lib\site-packages\pysolr.py", line 786, in add
return self._update(m, commit=commit, waitFlush=waitFlush, waitSearcher=waitSearcher)
File "C:\Python27\lib\site-packages\pysolr.py", line 379, in _update
return self._send_request('post', path, message, {'Content-type': 'text/xml; charset=utf-8'})
File "C:\Python27\lib\site-packages\pysolr.py", line 291, in _send_request
timeout=self.timeout)
File "C:\Python27\lib\site-packages\requests\api.py", line 87, in post
return request('post', url, data=data, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 279, in request
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies
)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 373, in send
r = adapter.send(request, **kwargs)
File "C:\Python27\lib\site-packages\requests\adapters.py", line 171, in send
timeout=timeout
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 421, in url
open
body=body, headers=headers)
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 273, in _ma
ke_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python27\lib\httplib.py", line 949, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 990, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 943, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 810, in _send_output
self.send(message_body)
File "C:\Python27\lib\httplib.py", line 775, in send
self.sock.sendall(str)
File "C:\Python27\lib\socket.py", line 222, in meth
return getattr(self._sock,name)(*args)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 143-149: ordinal not in range(
128)
I had the same error. It's seams to be fixed on master https://github.com/toastdriven/django-haystack. So you should use haystack 2.0 alpha/beta.