How can I optimize this Postgres query? - django

Question

I have a very slow query that I found in my logs and I don't know how to optimize it with an index.
This is the query and the explain:
SELECT
"myapp_image"."id",
"myapp_image"."deleted",
"myapp_image"."title",
"myapp_image"."subject_type",
"myapp_image"."data_source",
"myapp_image"."objects_in_field",
"myapp_image"."solar_system_main_subject",
"myapp_image"."description",
"myapp_image"."link",
"myapp_image"."link_to_fits",
"myapp_image"."image_file",
"myapp_image"."uploaded",
"myapp_image"."published",
"myapp_image"."updated",
"myapp_image"."watermark_text",
"myapp_image"."watermark",
"myapp_image"."watermark_position",
"myapp_image"."watermark_size",
"myapp_image"."watermark_opacity",
"myapp_image"."user_id",
"myapp_image"."plot_is_overlay",
"myapp_image"."is_wip",
"myapp_image"."size",
"myapp_image"."w",
"myapp_image"."h",
"myapp_image"."animated",
"myapp_image"."license",
"myapp_image"."is_final",
"myapp_image"."allow_comments",
"myapp_image"."moderator_decision",
"myapp_image"."moderated_when",
"myapp_image"."moderated_by_id",
"auth_user"."id",
"auth_user"."password",
"auth_user"."last_login",
"auth_user"."is_superuser",
"auth_user"."username",
"auth_user"."first_name",
"auth_user"."last_name",
"auth_user"."email",
"auth_user"."is_staff",
"auth_user"."is_active",
"auth_user"."date_joined",
"myapp_userprofile"."id",
"myapp_userprofile"."deleted",
"myapp_userprofile"."user_id",
"myapp_userprofile"."updated",
"myapp_userprofile"."real_name",
"myapp_userprofile"."website",
"myapp_userprofile"."job",
"myapp_userprofile"."hobbies",
"myapp_userprofile"."timezone",
"myapp_userprofile"."about",
"myapp_userprofile"."premium_counter",
"myapp_userprofile"."company_name",
"myapp_userprofile"."company_description",
"myapp_userprofile"."company_website",
"myapp_userprofile"."retailer_country",
"myapp_userprofile"."avatar",
"myapp_userprofile"."exclude_from_competitions",
"myapp_userprofile"."default_frontpage_section",
"myapp_userprofile"."default_gallery_sorting",
"myapp_userprofile"."default_license",
"myapp_userprofile"."default_watermark_text",
"myapp_userprofile"."default_watermark",
"myapp_userprofile"."default_watermark_size",
"myapp_userprofile"."default_watermark_position",
"myapp_userprofile"."default_watermark_opacity",
"myapp_userprofile"."accept_tos",
"myapp_userprofile"."receive_important_communications",
"myapp_userprofile"."receive_newsletter",
"myapp_userprofile"."receive_marketing_and_commercial_material",
"myapp_userprofile"."language",
"myapp_userprofile"."seen_realname",
"myapp_userprofile"."seen_email_permissions",
"myapp_userprofile"."signature",
"myapp_userprofile"."signature_html",
"myapp_userprofile"."show_signatures",
"myapp_userprofile"."post_count",
"myapp_userprofile"."autosubscribe",
"myapp_userprofile"."receive_forum_emails"
FROM "myapp_image"
LEFT OUTER JOIN "myapp_apps_iotd_iotd"
ON ("myapp_image"."id" = "myapp_apps_iotd_iotd"."image_id")
INNER JOIN "auth_user"
ON ("myapp_image"."user_id" = "auth_user"."id")
LEFT OUTER JOIN "myapp_userprofile"
ON ("auth_user"."id" = "myapp_userprofile"."user_id")
WHERE ("myapp_image"."is_wip" = FALSE
AND NOT ("myapp_image"."id" IN (SELECT
U0."id" AS Col1
FROM "myapp_image" U0
LEFT OUTER JOIN "myapp_apps_iotd_iotdvote" U1
ON (U0."id" = U1."image_id")
WHERE U1."id" IS NULL)
)
AND "myapp_apps_iotd_iotd"."id" IS NULL
AND "myapp_image"."id" < 372320
AND "myapp_image"."deleted" IS NULL)
ORDER BY "myapp_image"."id" DESC
LIMIT 1;
QUERY PLAN:
Limit (cost=36302.74..36302.75 rows=1 width=1143) (actual time=1922.839..1923.002 rows=1 loops=1)
-> Sort (cost=36302.74..36302.75 rows=1 width=1143) (actual time=1922.836..1922.838 rows=1 loops=1)
Sort Key: myapp_image.id DESC
Sort Method: top-N heapsort Memory: 26kB
-> Nested Loop Left Join (cost=17919.42..36302.73 rows=1 width=1143) (actual time=1332.216..1908.796 rows=3102 loops=1)
-> Nested Loop (cost=17919.14..36302.40 rows=1 width=453) (actual time=1332.195..1867.675 rows=3102 loops=1)
-> Hash Left Join (cost=17918.85..36302.09 rows=1 width=321) (actual time=1332.164..1815.315 rows=3102 loops=1)
Hash Cond: (myapp_image.id = myapp_apps_iotd_iotd.image_id)
Filter: (myapp_apps_iotd_iotd.id IS NULL)
Rows Removed by Filter: 722
-> Seq Scan on myapp_image (cost=17856.32..35882.67 rows=135958 width=321) (actual time=1329.110..1801.409 rows=3824 loops=1)
Filter: ((NOT is_wip) AND (deleted IS NULL) AND (NOT (hashed SubPlan 1)) AND (id < 372320))
Rows Removed by Filter: 305733
SubPlan 1
-> Gather (cost=1217.31..17856.31 rows=1 width=4) (actual time=36.399..680.882 rows=305712 loops=1)
Workers Planned: 2
Workers Launched: 2
-> Hash Left Join (cost=217.31..16856.21 rows=1 width=4) (actual time=52.855..536.185 rows=101904 loops=3)
Hash Cond: (u0.id = u1.image_id)
Filter: (u1.id IS NULL)
Rows Removed by Filter: 2509
-> Parallel Seq Scan on myapp_image u0 (cost=0.00..14672.82 rows=128982 width=4) (actual time=0.027..175.375 rows=103186 loops=3)
-> Hash (cost=123.25..123.25 rows=7525 width=8) (actual time=52.601..52.602 rows=7526 loops=3)
Buckets: 8192 Batches: 1 Memory Usage: 358kB
-> Seq Scan on myapp_apps_iotd_iotdvote u1 (cost=0.00..123.25 rows=7525 width=8) (actual time=0.038..33.074 rows=7526 loops=3)
-> Hash (cost=35.57..35.57 rows=2157 width=8) (actual time=3.013..3.015 rows=2157 loops=1)
Buckets: 4096 Batches: 1 Memory Usage: 117kB
-> Seq Scan on myapp_apps_iotd_iotd (cost=0.00..35.57 rows=2157 width=8) (actual time=0.014..1.480 rows=2157 loops=1)
-> Index Scan using auth_user_id_pkey on auth_user (cost=0.29..0.31 rows=1 width=132) (actual time=0.012..0.012 rows=1 loops=3102)
Index Cond: (id = myapp_image.user_id)
-> Index Scan using myapp_userprofile_user_id on myapp_userprofile (cost=0.29..0.33 rows=1 width=690) (actual time=0.008..0.008 rows=1 loops=3102)
Index Cond: (auth_user.id = user_id)
Planning time: 1.722 ms
Execution time: 1925.867 ms
(34 rows)
There is a long Seq Scan on myapp_image and I have tried adding the following index but it made it even slower:
create index on myapp_image using btree (is_wip, deleted, id);
What could be my optimization strategy?
This query is generated by the Django ORM and at this time I don't know yet what code path made it.

Answer 1

Based on generated query:
SELECT *
FROM "myapp_image"
LEFT OUTER JOIN "myapp_apps_iotd_iotd"
ON ("myapp_image"."id" = "myapp_apps_iotd_iotd"."image_id")
INNER JOIN "auth_user"
ON ("myapp_image"."user_id" = "auth_user"."id")
LEFT OUTER JOIN "myapp_userprofile"
ON ("auth_user"."id" = "myapp_userprofile"."user_id")
WHERE ("myapp_image"."is_wip" = false
AND NOT ("myapp_image"."id" IN (SELECT U0."id" AS Col1
FROM "myapp_image" U0
LEFT OUTER JOIN "myapp_apps_iotd_iotdvote" U1
ON (U0."id" = U1."image_id")
WHERE U1."id" IS NULL))
AND "myapp_apps_iotd_iotd"."id" IS NULL
AND "myapp_image"."id" < 372320
AND "myapp_image"."deleted" IS NULL)
ORDER BY "myapp_image"."id" DESC
LIMIT 1;
I propose to add index:
create index on myapp_image using btree (id, is_wip, deleted);
-- id as a first column