Related
I was trying to run a hyperparameter tuning job on Vertex AI using the Python SDK described here. Around 2 hours ago it was successful sending the job to run. I noticed there are some errors in my code so the run failed, and I went back and fix it, and I re-run the code and what I got was as below.
Traceback (most recent call last):
File "/workspace/.pip-modules/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 67, in error_remapped_callable
return callable_(*args, **kwargs)
File "/workspace/.pip-modules/lib/python3.8/site-packages/grpc/_channel.py", line 946, in __call__
return _end_unary_response_blocking(state, call, False, None)
File "/workspace/.pip-modules/lib/python3.8/site-packages/grpc/_channel.py", line 849, in _end_unary_response_blocking
raise _InactiveRpcError(state)
grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
status = StatusCode.INTERNAL
details = "Internal error encountered."
debug_error_string = "{"created":"#1623393121.374988331","description":"Error received from peer ipv4:142.251.33.74:443","file":"src/core/lib/surface/call.cc","file_line":1066,"grpc_message":"Internal error encountered.","grpc_status":13}"
>
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/workspace/ariel_ml_2021/hparam_tuning.py", line 140, in <module>
create_hyperparameter_tuning_job_python_package()
File "/workspace/ariel_ml_2021/hparam_tuning.py", line 133, in create_hyperparameter_tuning_job_python_package
response = client.create_hyperparameter_tuning_job(
File "/workspace/.pip-modules/lib/python3.8/site-packages/google/cloud/aiplatform_v1/services/job_service/client.py", line 1363, in create_hyperparameter_tuning_job
response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
File "/workspace/.pip-modules/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py", line 145, in __call__
return wrapped_func(*args, **kwargs)
File "/workspace/.pip-modules/lib/python3.8/site-packages/google/api_core/grpc_helpers.py", line 69, in error_remapped_callable
six.raise_from(exceptions.from_grpc_error(exc), exc)
File "<string>", line 3, in raise_from
google.api_core.exceptions.InternalServerError: 500 Internal error encountered.
I thought this might be due to my changes in the python code that causes the error, so I try with the original copy (without making any changes), and the error persists. The code for hyperparameter tuning is as below if required.
from google.cloud import aiplatform
def create_hyperparameter_tuning_job_python_package(
project: str = "<my_project_id>",
display_name: str = "<some_description>",
executor_image_uri: str = "us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-4:latest",
package_uri: str = "gs://<bucket_name>/",
python_module: str = "train_second", # located at gs://<bucket_name>/train_second.py
location: str = "us-central1",
api_endpoint: str = "us-central1-aiplatform.googleapis.com",
):
client_options = {"api_endpoint": api_endpoint}
client = aiplatform.gapic.JobServiceClient(client_options=client_options)
metric = {
"metric_id": "ariel_score",
"goal": aiplatform.gapic.StudySpec.MetricSpec.GoalType.MAXIMIZE,
}
conditional_param_H1 = {
"parameter_spec": {
"parameter_id": "H1",
"discrete_value_spec": {"values": [4, 8, 16, 32, 64, 128, 256, 512, 1024]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_H2 = {
"parameter_spec": {
"parameter_id": "H2",
"discrete_value_spec": {"values": [64, 128, 256, 512, 1024]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_H3 = {
"parameter_spec": {
"parameter_id": "H3",
"discrete_value_spec": {"values": [4, 8, 16, 32, 64, 128, 256, 512, 1024]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_D1 = {
"parameter_spec": {
"parameter_id": "D1",
"double_value_spec": {"min_value": 0.01, "max_value": 0.5},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_mean = {
"parameter_spec": {
"parameter_id": "mean",
"discrete_value_spec": {"values": [0., 1.]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_std = {
"parameter_spec": {
"parameter_id": "std",
"double_value_spec": {"min_value": 0.005, "max_value": 0.5},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
conditional_param_lr = {
"parameter_spec": {
"parameter_id": "lr",
"discrete_value_spec": {"values": [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
},
"parent_discrete_values": {"values": [10, 25, 50, 100]}
}
parameter = {
"parameter_id": "batch_size",
"discrete_value_spec": {"values": [10, 25, 50, 100]},
"scale_type": aiplatform.gapic.StudySpec.ParameterSpec.ScaleType.UNIT_LINEAR_SCALE,
"conditional_parameter_specs": [
conditional_param_H1,
conditional_param_H2,
conditional_param_H3,
conditional_param_D1,
conditional_param_mean,
conditional_param_std,
conditional_param_lr,
],
}
# Trial job spec
machine_spec = {
"machine_type": "e2-standard-4",
}
worker_pool_spec = {
"machine_spec": machine_spec,
"replica_count": 1,
"python_package_spec": {
"executor_image_uri": executor_image_uri,
"package_uris": [package_uri],
"python_module": python_module,
"args": [],
}
}
# hparam tuning job
hyperparameter_tuning_job = {
"display_name": display_name,
"max_trial_count": 2,
"parallel_trial_count": 2,
"study_spec": {
"metrics": [metric],
"parameters": [parameter],
},
"trial_job_spec": {"worker_pool_specs": [worker_pool_spec]},
}
parent = f"projects/{project}/locations/{location}"
response = client.create_hyperparameter_tuning_job(
parent=parent, hyperparameter_tuning_job=hyperparameter_tuning_job
)
print(f"response:", response)
if __name__ == "__main__":
create_hyperparameter_tuning_job_python_package()
Thanks in advance.
It seems that the endpoint at us-central1 is experiencing the problem. The workaround is to use another endpoint such as with us-east1 and the problem is solved.
Your package_uri is incorrect. It should point to a file containing the Python package (i.e. a tar.bz file with all the code), not a directory or a bucket.
i try to digest an array of bytes with SHA512
Java:
MessageDigest sha512 = MessageDigest.getInstance("SHA-512");
System.out.println("sha512: "+Arrays.toString(sha512.digest(buffer.array())));
this sample computes with input:
[18, 52, 49, 114, -101, -17, 46, -74, -108, 84, 7, -37, -51, -87, -75, 123, 113, 71, -52, 109, -69, 6, 46, 19, -108, 100, -33, 14, 74, 122, -126, -103, 81, -93]
this output:
[-6, -75, -91, -50, 3, 5, -78, -74, -63, -33, -103, 24, -18, 39, -45, -22, 30, -10, 58, -7, -2, -28, 77, 43, -78, 58, 123, -101, -12, 22, 63, 16, -56, -17, 58, 13, -26, 61, -45, -22, 100, 121, -118, -86, 53, 115, -42, 90, -70, 67, -61, 100, 54, -46, -113, -119, -95, 27, 23, -95, -76, 51, 52, 99]
how can i get the same result using CryptoJS on postman?
thanks-
Here ist the java code, that i want translate to javaScript usung CryptoJS into postman as pre-script to compute a SHA512 hash:
package hashtest;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import com.google.common.io.BaseEncoding;
public class SpinHash
{
private static byte[] applyChallenge(byte[] encrypted, byte[] challenge) throws NoSuchAlgorithmException {
byte[] KEY_CHALLENGE_SEPARATOR = "".getBytes();
MessageDigest sha512 = MessageDigest.getInstance("SHA-512");
ByteBuffer buffer = ByteBuffer.allocate(encrypted.length + challenge.length + KEY_CHALLENGE_SEPARATOR.length);
buffer.put(encrypted);
buffer.put(KEY_CHALLENGE_SEPARATOR);
buffer.put(challenge);
buffer.compact();
return sha512.digest(buffer.array());
}
private static String returnHashPin(String spin, String challenge) throws NoSuchAlgorithmException
{
BaseEncoding BASE16 = BaseEncoding.base16();
byte[] _challenge = BASE16.decode(challenge.toUpperCase());
byte[] _SPIN = BASE16.decode(spin.toUpperCase());
byte[] answer;
answer = applyChallenge(_SPIN, _challenge);
return BASE16.encode(answer).toUpperCase();
}
public static void main(String[] args) throws NoSuchAlgorithmException
{
System.out.println( "Hash-Wert: " + returnHashPin("1234","089C25A81BA2B17601FB89A47CEE561737AA41055AFE88766883CA9C9E7545F5") );
}
}
I feel like there should be an easy way to do this but I can not find a solution online. I'm looking for a one liner something pythonic.
I know you can do this:
Img[:,:,2]=200
but how can you add a condition to this?
if Img[:,:,2]<=100: 200
or
image =[i==100 for i in Img[:,:,2] if i <= 100]
Thanks
To simulate if a[:,:,2]<=100: 200, we can use np.where -
a[:,:,2] = np.where(a[:,:,2] <= 100, 200, a[:,:,2])
Or simply use the mask to assign -
a[a[:,:,2] <= 100,2] = 200
Sample run -
In [379]: a
Out[379]:
array([[[ 78, 134, 7],
[154, 37, 146],
[ 39, 95, 13]],
[[114, 138, 100],
[175, 198, 148],
[ 39, 130, 37]]])
In [380]: a[a[:,:,2] <= 100,2] = 200
In [381]: a
Out[381]:
array([[[ 78, 134, 200],
[154, 37, 146],
[ 39, 95, 200]],
[[114, 138, 200],
[175, 198, 148],
[ 39, 130, 200]]])
Example creation of Dictionary: Dictionary processes Elements in arbitrary fashion. Hence I get the output in mixed order which doesnt correlate with my list x.
x = [100,2,300,4,75]
dct = {}
for i in x:
dct['lst_%s' % i] = []
for z in range(0,5):
dct['lst_%s' % i].extend((x[z],x[z-1]))
print(dct)
{'lst_300': [100, 75, 2, 100, 300, 2, 4, 300, 75, 4], 'lst_75': [100, 75, 2, 100, 300, 2, 4, 300, 75, 4], 'lst_100': [100, 75, 2, 100, 300, 2, 4, 300, 75, 4], 'lst_2': [100, 75, 2, 100, 300, 2, 4, 300, 75, 4], 'lst_4': [100, 75, 2, 100, 300, 2, 4, 300, 75, 4]}
Using sorted key I do get dictionary headers in order corresponding to my list x but how can I retrieve the elements for the same ?
x = [100,2,300,4,75]
dct = {}
for i in x:
dct['lst_%s' % i] = []
for z in range(0,5):
dct['lst_%s' % i].extend((x[z],x[z-1]))
print(sorted(dct))
['lst_100', 'lst_2', 'lst_300', 'lst_4', 'lst_75']
dict is unordered by definition. If you really want an ordered map, use collections.OrderedDict
I have a list:
def clc = [[1, 15, 30, 42, 48, 100], [58, 99], [16, 61, 85, 96, 98], [2, 63, 84, 90, 91, 97], [16, 61, 85, 96], [23, 54, 65, 95], [16, 29, 83, 94], [0, 31, 42, 93], [33, 40, 51, 56, 61, 62, 64, 89, 92], [0, 63, 84, 90, 91]]
and a sublist
def subclc = [[1, 15, 30, 42, 48, 100], [58, 99], [16, 61, 85, 96, 98], [2, 63, 84, 90, 91, 97]]
I need to remove sublist from original list
I do so:
subclc.each{
clc.remove(it)
}
but it throws an exceprion Exception in thread "main" java.util.ConcurrentModificationException
I do not understand where is problem and how to solve the problem
Short answers:
For more groovyness and immutability, preserving the original list:
def removed = clc - subclc
assert removed == [[16, 61, 85, 96], [23, 54, 65, 95], [16, 29, 83, 94], [0, 31, 42, 93], [33, 40, 51, 56, 61, 62, 64, 89, 92], [0, 63, 84, 90, 91]]
And the java way, changing the original list:
clc.removeAll subclc
assert clc == [[16, 61, 85, 96], [23, 54, 65, 95], [16, 29, 83, 94], [0, 31, 42, 93], [33, 40, 51, 56, 61, 62, 64, 89, 92], [0, 63, 84, 90, 91]]
Long answer:
You are Iterator-ing through a list while changing the list. In this case you are better using Iterator.remove(), which is being abstracted by the foreach loop.
When you change the list using the foreach loop, you bump into the iterator checks for modification using checkForComodification(). Getting the iterator explicitly works:
list1 = [10,20,30,40,50,60,70,80,90]
list2 = [50,60,80]
def iter = list1.iterator()
while (iter.hasNext()) {
def item = iter.next()
if (list2.contains(item)) iter.remove()
}
assert list1 == [10,20,30,40,70,90]
Or you can use indexes. Note you need to control the index:
list1 = [10,20,30,40,50,60,70,80,90]
list2 = [50,60,80]
for (int i = 0; i < list1.size(); i++) {
def item = list1[i]
if (list2.contains(item)) { list1.remove i-- }
}
assert list1 == [10,20,30,40,70,90]