I am doing an exercise where the goal it's to match my current calendar with other users.
To do this, I created a UserProfile App and Schedule App. Each user has a profile that can have multiple intervals.
Considering my current calendar:
{
"count": 1,
"next": null,
"previous": null,
"results": [
{
"id": 3,
"user": {
"id": 3,
"username": "john.doe",
"first_name": "John",
"last_name": "Doe"
},
"calendar": [
{
"id": 1,
"mon": true,
"tue": true,
"wed": true,
"thu": true,
"fri": true,
"sat": true,
"sun": true,
"start_date": "09:30",
"end_date": "12:20"
},
{
"id": 2,
"mon": true,
"tue": true,
"wed": true,
"thu": true,
"fri": true,
"sat": true,
"sun": true,
"start_date": "14:00",
"end_date": "23:00"
}
]
}
]}
When I am doing a call to the endpoint /api/search/users it returns all User Profiles with info from each user.
example:
{
"count": 99,
"next": "http://localhost:8000/api/search/users?page=2",
"previous": null,
"results": [
{
"id": 1,
"user": {
"id": 1,
"username": "john.bender.99",
"first_name": "John",
"last_name": "Bender"
},
"calendar": [
{
"id": 2,
"mon": true,
"tue": true,
"wed": true,
"thu": false,
"fri": true,
"sat": false,
"sun": false,
"start_date": "09:30",
"end_date": "12:20"
},
{
"id": 55,
"mon": false,
"tue": true,
"wed": true,
"thu": false,
"fri": true,
"sat": false,
"sun": false,
"start_date": "14:30",
"end_date": "19:20"
}
]
}
]}
Now, what I want to do actually is a search for related users with my calendar to know what days/hours we have a match.
When I do a call to this endpoint /api/search/users?related=self, I want to see this
{
"count": 2,
"results": [
{
"id": 87,
"user": {
"id": 87,
"username": "diana.taller",
"first_name": "Diana",
"last_name": "Taller"
},
"calendar": [
{
"id": 2,
"mon": true,
"tue": true,
"wed": true,
"thu": false,
"fri": true,
"sat": false,
"sun": false,
"start_date": "10:30",
"end_date": "11:20"
},
{
"id": 55,
"mon": false,
"tue": true,
"wed": true,
"thu": false,
"fri": true,
"sat": false,
"sun": false,
"start_date": "16:30",
"end_date": "17:20"
}
]
},{
"id": 128,
"user": {
"id": 128,
"username": "therockjosh",
"first_name": "Josh",
"last_name": "Bail"
},
"calendar": [
{
"id": 2,
"mon": false,
"tue": false,
"wed": false,
"thu": false,
"fri": true,
"sat": false,
"sun": false,
"start_date": "10:30",
"end_date": "11:20"
},
{
"id": 55,
"mon": false,
"tue": false,
"wed": false,
"thu": false,
"fri": true,
"sat": true,
"sun": true,
"start_date": "14:30",
"end_date": "17:00"
}
]
}
]}
The interception between my availability and from users is done between per day and then each interval to see when we have a match.
Inside my Search App, I created this
if related == "self":
current_user_profile = UserProfile.objects.filter(user=self.request.user)
related_users = UserProfile.objects.filter(calendar__in=current_user_profile.calendar.all())
return related_users
If I call current_user_profile, returns me the current user data as I provided you before.
If I call UserProfile.objects.all(), returns me the user's data as I provided you before.
But for some reason, I can't call calendar from current_user_profile.calendar as this image shows.
Is anyone have some idea how could I do this?
I think you need to use get function if you wanna get the object.
if related == "self":
# not UserProfile.objects.filter in order to get the UserProfile object.
current_user_profile = UserProfile.objects.get(user=self.request.user)
related_users = UserProfile.objects.filter(calendar__in=current_user_profile.calendar.all())
return related_users
Here we have the solution I found to exclude my user from the search.
current_user_profile = UserProfile.objects.get(user=self.request.user)
related_users = UserProfile.objects\
.filter(calendar__in=current_user_profile.calendar.all()) \
.exclude(user_id=current_user_profile.id)
return related_users
Related
I want to extract id value from the following response body.
[
{
"id": "6311ebc04318ce16bedd1a14",
"creationMethod": "board-creation",
"name": "workspace18669413",
"credits": [],
"displayName": "Prashant Vadher's workspace",
"desc": "",
"descData": {
"emoji": {}
},
"domainName": "epam.com",
"idBoards": [],
"idEnterprise": null,
"idMemberCreator": "6310461591394a03d2012ca3",
"invited": false,
"invitations": [],
"limits": {
"orgs": {
"totalMembersPerOrg": {
"status": "ok",
"disableAt": 4000,
"warnAt": 3200
},
"freeBoardsPerOrg": {
"status": "ok",
"disableAt": 10,
"warnAt": 3
}
}
},
"memberships": [
{
"idMember": "6310461591394a03d2012ca3",
"memberType": "admin",
"unconfirmed": false,
"deactivated": false,
"id": "6311ebc04318ce16bedd1a16"
}
],
"membersCount": 1,
"prefs": {
"permissionLevel": "private",
"orgInviteRestrict": [],
"boardInviteRestrict": "any",
"externalMembersDisabled": false,
"associatedDomain": null,
"googleAppsVersion": 1,
"boardVisibilityRestrict": {
"private": "org",
"org": "org",
"enterprise": "org",
"public": "org"
},
"boardDeleteRestrict": {
"private": "org",
"org": "org",
"enterprise": "org",
"public": "org"
},
"attachmentRestrictions": null
},
"powerUps": [],
"products": [],
"billableMemberCount": 1,
"activeBillableMemberCount": 1,
"billableCollaboratorCount": 0,
"url": "https://trello.com/workspace18669413",
"website": null,
"logoHash": null,
"logoUrl": null,
"premiumFeatures": [
"additionalBoardBackgrounds",
"additionalStickers",
"customBoardBackgrounds",
"customEmoji",
"customStickers",
"plugins"
],
"promotions": [],
"enterpriseJoinRequest": {},
"standardVariation": null,
"availableLicenseCount": null,
"maximumLicenseCount": null,
"ixUpdate": null,
"teamType": null,
"canEditDomain": true
},
{
"id": "6310461e34b26a003d4f65d9",
"creationMethod": null,
"name": "userworkspace78017157",
"credits": [],
"displayName": "Trello Workspace",
"desc": "",
"descData": {
"emoji": {}
},
"domainName": "epam.com",
"idBoards": [
"6311d55c9640b1010ee3a9e6",
"6311e8216b6cd7003e656e5c",
"6311ec42e22f75004513983e",
"6311f1b3b36e0c007e68ccbb"
],
"idEnterprise": null,
"idMemberCreator": "6310461591394a03d2012ca3",
"invited": false,
"invitations": [],
"limits": {
"orgs": {
"totalMembersPerOrg": {
"status": "ok",
"disableAt": 4000,
"warnAt": 3200
},
"freeBoardsPerOrg": {
"status": "warn",
"disableAt": 10,
"warnAt": 3,
"count": 3
}
}
},
"memberships": [
{
"idMember": "6310461591394a03d2012ca3",
"memberType": "admin",
"unconfirmed": false,
"deactivated": false,
"id": "6310461e34b26a003d4f65db"
}
],
"membersCount": 1,
"prefs": {
"permissionLevel": "private",
"orgInviteRestrict": [],
"boardInviteRestrict": "any",
"externalMembersDisabled": false,
"associatedDomain": null,
"googleAppsVersion": 1,
"boardVisibilityRestrict": {
"private": "org",
"org": "org",
"enterprise": "org",
"public": "org"
},
"boardDeleteRestrict": {
"private": "org",
"org": "org",
"enterprise": "org",
"public": "org"
},
"attachmentRestrictions": null
},
"powerUps": [],
"products": [],
"billableMemberCount": 1,
"activeBillableMemberCount": 1,
"billableCollaboratorCount": 0,
"url": "https://trello.com/userworkspace78017157",
"website": null,
"logoHash": null,
"logoUrl": null,
"premiumFeatures": [
"additionalBoardBackgrounds",
"additionalStickers",
"customBoardBackgrounds",
"customEmoji",
"customStickers",
"plugins"
],
"promotions": [],
"enterpriseJoinRequest": {},
"standardVariation": null,
"availableLicenseCount": null,
"maximumLicenseCount": null,
"ixUpdate": "6",
"teamType": null,
"canEditDomain": true
}
]
I have tried following code:
*jsonData = JSON.parse(responseBody)
console.log("JSON Body - " + JSON.stringify(jsonData))
pm.environment.set('idOrganization', JSON.stringify(jsonData.id))
console.log("idOrganization: " + JSON.stringify(jsonData.id))*
But getting undefined value in console.
idOrganization: undefined
Is there any other way to save this value in environment variable ?
could you try such:
const responseJson = pm.response.json();
pm.environment.set("idOrganization", responseJson[0]["id"]);
this is for the first item from your array.
Guess your requirement is unclear, there are two id's present, which one you want to retrieve? If any specific it is already answered.
If you want both with comma separated, then please try this.
const responseBODY = pm.response.json();
let idValue = "";
for (let i=0; i < responseBODY.length; i++) {
idValue += responseBODY[i].id+",";
}
finalIDValue = idValue.substring(0,idValue.length - 1);
//put your logic how you want to use it
pm.environment.set("idOrganization", finalIDValue);
my Comment model is like below:
class Comment(models.Model):
user = models.ForeignKey(User)
content = models.TextField()
parent = models.ForeignKey('self', related_name='answers')
I want to serializer comment and all of its answers. So i've wrote this serializer:
class CommentSerializer(ModelSerializer):
user = UserSerializer(fields=('get_full_name',), allow_null=True)
class Meta:
model = Comment
fields = ['user', 'content', 'answers']
depth = 10
now when I serialize a comment by this serializer, it works properly just for level 1 of the comment. the answers object shows all the data, such as user's password! or it shows 'parent' while I didn't include parent to the fields of Meta class of the serializer.
the following is the output of this serializer:
{
"data": {
"pk": 15,
"user": null,
"name": "rwqerweqr",
"email": null,
"content": "A",
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"answers": [
{
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": 3,
"parent": 15
}
}
}
}
}
}
}
}
}
},
{
"id": 16,
"content": "D",
"name": "werwer",
"email": null,
"approved": false,
"created_at": "2019-04-01T18:37:46.981327+04:30",
"user": null,
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 15,
"content": "A",
"name": "rwqerweqr",
"email": null,
"approved": false,
"created_at": "2019-03-31T23:16:44.980338+04:30",
"user": null,
"parent": {
"id": 13,
"content": "C",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T21:27:29.152248+04:30",
"user": {
"id": 3,
"is_superuser": true,
"email": "email#gmail.com",
"phone": "+111111111111",
"password": "pbkdf2_sha256$36000$RDdFKTXhvzhb$GaR2Y0p1DBp/5pO5yqayMmdNN/XjnoWEdKshYbmkwCk=",
"first_name": "Alex",
"last_name": "Doe",
"created_at": "2019-03-26T21:16:53.818407+04:30",
"last_login": "2019-03-26T21:17:14.987512+04:30",
"is_active": true,
"is_staff": true,
"groups": [],
"user_permissions": []
},
"parent": {
"id": 14,
"content": "B",
"name": null,
"email": null,
"approved": false,
"created_at": "2019-03-31T23:08:13.374350+04:30",
"user": 3,
"parent": 15
}
}
}
}
}
}
}
}
}
}
]
}
}
as you see, in the answers object, it shows user's data completely, include of its hashed password or permissions.
or it shows parent field while it is not in the serializer at all!.
how to solve this problem?
thanks.
UPDATE
usecase: Comments B and D are answers of comment A. (b.parent == A)
As displayed in django-rest-framework docs:
The depth option should be set to an integer value that indicates the depth of relationships that should be traversed before reverting to a flat representation.
If you want to customize the way the serialization is done you'll need to define the field yourself.
You should serialize the answers field, in order to hide those informations. I would follow the approach of SerializerMethodField. I will write an example:
class CommentSerializer(ModelSerializer):
user = UserSerializer(fields=('get_full_name',), allow_null=True)
answers = serializers.SerializerMethodField()
class Meta:
model = Comment
fields = ['user', 'content', 'answers']
def get_answers(self, instance):
# Here you will create your custom serialization
I have the following JSON stream coming from Twitter.
{
"created_at": "Thu Sep 27 21:02:00 +0000 2018",
"id": 1045418301336244224,
"id_str": "1045418301336244224",
"text": "Conditional Branching Now Supported in AWS Systems Manager Automation - #awscloud #amazon #aws",
"source": "Buffer",
"truncated": false,
"in_reply_to_status_id": null,
"in_reply_to_status_id_str": null,
"in_reply_to_user_id": null,
"in_reply_to_user_id_str": null,
"in_reply_to_screen_name": null,
"user": {
"id": 14687423,
"id_str": "14687423",
"name": "Casey Becking",
"screen_name": "caseybecking",
"location": "Huntington Beach, CA",
"url": "http://caseybecking.com",
"description": "I do stuff with computers for #rackspace , geek at heart! play and watch to much hockey, someday I'll make a personal website.",
"translator_type": "none",
"protected": false,
"verified": false,
"followers_count": 4191,
"friends_count": 2412,
"listed_count": 90,
"favourites_count": 794,
"statuses_count": 12995,
"created_at": "Wed May 07 15:03:23 +0000 2008",
"utc_offset": null,
"time_zone": null,
"geo_enabled": true,
"lang": "en",
"contributors_enabled": false,
"is_translator": false,
"profile_background_color": "000000",
"profile_background_image_url": "http://abs.twimg.com/images/themes/theme15/bg.png",
"profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme15/bg.png",
"profile_background_tile": false,
"profile_link_color": "ABB8C2",
"profile_sidebar_border_color": "000000",
"profile_sidebar_fill_color": "000000",
"profile_text_color": "000000",
"profile_use_background_image": false,
"profile_image_url": "http://pbs.twimg.com/profile_images/981617292546060289/RMX0GQFe_normal.jpg",
"profile_image_url_https": "https://pbs.twimg.com/profile_images/981617292546060289/RMX0GQFe_normal.jpg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/14687423/1439137746",
"default_profile": false,
"default_profile_image": false,
"following": null,
"follow_request_sent": null,
"notifications": null
},
"geo": null,
"coordinates": null,
"place": null,
"contributors": null,
"is_quote_status": false,
"quote_count": 0,
"reply_count": 0,
"retweet_count": 0,
"favorite_count": 0,
"entities": {
"hashtags": [{
"text": "amazon",
"indices": [106, 113]
}, {
"text": "aws",
"indices": [114, 118]
}],
"urls": [{
"url": "",
"expanded_url": "https://buff.ly/2zwRyBx",
"display_url": "buff.ly/2zwRyBx",
"indices": [72, 95]
}],
"user_mentions": [{
"screen_name": "awscloud",
"name": "Amazon Web Services",
"id": 66780587,
"id_str": "66780587",
"indices": [96, 105]
}],
"symbols": []
},
"favorited": false,
"retweeted": false,
"possibly_sensitive": false,
"filter_level": "low",
"lang": "en",
"timestamp_ms": "1538082120628",
"emoticons": [],
"sentiments": "Neutral"
}
How do I parse, analyze and process this JSON using Kinesis Analytics?
The arrays should be flattened and this is very doable in Hive but need to do the same in Kinesis Analytics.
HI all we are migrating out database from on premises to Amazon aurora.our database size is around 136GB moreover few tables have over millions of records each. Howover after full load complete out of millions rows approx 200,000 to 300,000 rows gets migrated.WE dont know where we are falling since we are new to DMS.Can anyone know how can we migrate exact count of rows.
migration type :full load
Here are our AWS DMS task settings
{
"TargetMetadata": {
"TargetSchema": "",
"SupportLobs": true,
"FullLobMode": true,
"LobChunkSize": 64,
"LimitedSizeLobMode": false,
"LobMaxSize": 0,
"LoadMaxFileSize": 0,
"ParallelLoadThreads": 0,
"BatchApplyEnabled": false
},
"FullLoadSettings": {
"FullLoadEnabled": true,
"ApplyChangesEnabled": false,
"TargetTablePrepMode": "TRUNCATE_BEFORE_LOAD",
"CreatePkAfterFullLoad": false,
"StopTaskCachedChangesApplied": false,
"StopTaskCachedChangesNotApplied": false,
"ResumeEnabled": false,
"ResumeMinTableSize": 100000,
"ResumeOnlyClusteredPKTables": true,
"MaxFullLoadSubTasks": 15,
"TransactionConsistencyTimeout": 600,
"CommitRate": 10000
},
"Logging": {
"EnableLogging": true,
"LogComponents": [
{
"Id": "SOURCE_UNLOAD",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "SOURCE_CAPTURE",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TARGET_LOAD",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TARGET_APPLY",
"Severity": "LOGGER_SEVERITY_DEFAULT"
},
{
"Id": "TASK_MANAGER",
"Severity": "LOGGER_SEVERITY_DEFAULT"
}
],
"CloudWatchLogGroup": "dms-tasks-krishna-smartdata",
"CloudWatchLogStream": "dms-task-UERQWLR6AYHYIEKMR3HN2VL7T4"
},
"ControlTablesSettings": {
"historyTimeslotInMinutes": 5,
"ControlSchema": "",
"HistoryTimeslotInMinutes": 5,
"HistoryTableEnabled": true,
"SuspendedTablesTableEnabled": true,
"StatusTableEnabled": true
},
"StreamBufferSettings": {
"StreamBufferCount": 3,
"StreamBufferSizeInMB": 8,
"CtrlStreamBufferSizeInMB": 5
},
"ChangeProcessingDdlHandlingPolicy": {
"HandleSourceTableDropped": true,
"HandleSourceTableTruncated": true,
"HandleSourceTableAltered": true
},
"ErrorBehavior": {
"DataErrorPolicy": "LOG_ERROR",
"DataTruncationErrorPolicy": "LOG_ERROR",
"DataErrorEscalationPolicy": "SUSPEND_TABLE",
"DataErrorEscalationCount": 0,
"TableErrorPolicy": "SUSPEND_TABLE",
"TableErrorEscalationPolicy": "STOP_TASK",
"TableErrorEscalationCount": 0,
"RecoverableErrorCount": -1,
"RecoverableErrorInterval": 5,
"RecoverableErrorThrottling": true,
"RecoverableErrorThrottlingMax": 1800,
"ApplyErrorDeletePolicy": "IGNORE_RECORD",
"ApplyErrorInsertPolicy": "LOG_ERROR",
"ApplyErrorUpdatePolicy": "LOG_ERROR",
"ApplyErrorEscalationPolicy": "LOG_ERROR",
"ApplyErrorEscalationCount": 0,
"FullLoadIgnoreConflicts": true
},
"ChangeProcessingTuning": {
"BatchApplyPreserveTransaction": true,
"BatchApplyTimeoutMin": 1,
"BatchApplyTimeoutMax": 30,
"BatchApplyMemoryLimit": 500,
"BatchSplitSize": 0,
"MinTransactionSize": 1000,
"CommitTimeout": 1,
"MemoryLimitTotal": 1024,
"MemoryKeepTime": 60,
"StatementCacheSize": 50
}
}
Mapping Method:
{
"rules": [
{
"rule-type": "selection",
"rule-id": "1",
"rule-name": "1",
"object-locator": {
"schema-name": "dbo",
"table-name": "%"
},
"rule-action": "include"
},
{
"rule-type": "transformation",
"rule-id": "2",
"rule-name": "2",
"rule-target": "schema",
"object-locator": {
"schema-name": "dbo"
},
"rule-action": "rename",
"value": "smartdata_int"
}
]
}
You should have the option of setting up CloudWatch logs for each DMS task. Have you inspected the logs for this task? Do you have varchar/text columns > 32KB? These will be truncated when migrating data into a target like redshift, so be aware that this will count towards your error count.
First thing to do is to increase log level :
"Logging": {
"EnableLogging": true,
"LogComponents": [{
"Id": "SOURCE_UNLOAD",
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
},{
"Id": "SOURCE_CAPTURE",
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
},{
"Id": "TARGET_LOAD",
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
},{
"Id": "TARGET_APPLY",
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
},{
"Id": "TASK_MANAGER",
"Severity": "LOGGER_SEVERITY_DETAILED_DEBUG"
}]
},
Then you will be able to get details about errors occuring.
Turn on validation:
https://docs.aws.amazon.com/dms/latest/userguide/CHAP_Validating.html
This will slow the migration down so you cloud also look at splitting this out into multiple tasks and running them on multiple replication instances, expand rule 1 out into multiple rules, rather than '%' add a condition that meets a subset of the tables.
You might also try a different replication engine, 3.1.1 has just been released, at the time of writing there are no release notes for 3.1.1.
https://docs.aws.amazon.com/dms/latest/userguide/CHAP_ReleaseNotes.html
I am new to elasticsearch. I am trying to retrieve the selected fields from nested object returned by elasticsearch. Below is the object stored in elasticsearch index:
{
"_index": "xxx",
"_type": "user",
"_id": "2",
"_version": 1,
"exists": true,
"_source": {
"user": {
"user_auth": {
"username": "nickcoolster#gmail.com",
"first_name": "",
"last_name": "",
"is_active": false,
"_state": {
"adding": false,
"db": "default"
},
"id": 2,
"is_superuser": false,
"is_staff": false,
"last_login": "2012-07-10 21:11:53",
"password": "sha1$a6caa$cba2f821678ccddc4d70c8bf0c8e0655ab5c279b",
"email": "nickcoolster#gmail.com",
"date_joined": "2012-07-10 21:11:53"
},
"user_account": {},
"user_profile": {
"username": null,
"user_id": 2,
"following_count": 0,
"sqwag_count": 0,
"pwd_reset_key": null,
"_state": {
"adding": false,
"db": "default"
},
"personal_message": null,
"followed_by_count": 0,
"displayname": "nikhil11",
"fullname": "nikhil11",
"sqwag_image_url": null,
"id": 27,
"sqwag_cover_image_url": null
}
}
}
}
Now I want only certain fields to be returned from user.user_auth(like password,superuser etc should not be returned).
I am using django PyES and below is the code that i tried:
fields = ["user.user_auth.username","user.user_auth.first_name","user.user_auth.last_name","user.user_auth.email"]
result = con.search(query=q,indices="xxx",doc_types="user",fields=fields)
but the result that I get is only email being retrieved (i:e only last field being returned):
{
"user.user_auth.email": "nikhiltyagi.eng#gmail.com"
}
I want this abstraction for both the nested objects (i:e user_auth,user_profile)
how do I do this?
What about using latest django-haystack? It covers also ElasticSearch as a backend so you can get well binded ElasticSearch FTS to your project.