ERROR: Teradata execute: Numeric overflow occurred during computation - sas

I've got this error: Teradata execute: Numeric overflow occurred during computation.
I have read the similar questions on this forum but still can't understand what should I do to resolve the problem.
I supposed that the reason is that I have computation on step when i devide NATIONAL_AMOUNT on DEBT_EXPENSE.
After using CAST there was no resolving still.
Has anyone thoughts? Please help me.
I think that the problem is here:
EXECUTE (create multiset volatile table Debt_&mon. AS
(SELECT
t1.TABLE_BUSINESS_MONTH,
t1.REGION,
t1.SEGMENT,
t1.ACC_METH,
COUNT(t1.ACC_N) as Count_ACC_N,
SUM(t1.NATIONAL_UNPAID) AS NATIONAL_UNPAID,
SUM(t1.DEBT_EXPENSE) as DEBT_EXPENSE,
SUM(t2.NATIONAL_AMOUNT) as NATIONAL_AMOUNT,
(case when SUM(t2.NATIONAL_AMOUNT)=0 then 0 else (cast(SUM(t1.DEBT_EXPENSE) as Decimal(20,6))/cast(SUM(t2.NATIONAL_AMOUNT) as Decimal(20,6)) end) as 'РСДЗ/Д'n,
(case when SUM(t1.DEBT_EXPENSE)=0 then 0 else cast(SUM(t2.NATIONAL_AMOUNT) as Decimal(20,6))/cast(SUM(t1.DEBT_EXPENSE) as Decimal(20,6)) end) as 'Д/ДЗ'n
FROM Prev_Debt_&mon. t1
LEFT JOIN CLC_&mon. t2 on
t1.REGION=t2.REGION and
t1.SEGMENT=t2.SEGMENT and
t1.ACC_METH=t2.ACC_METH
GROUP BY t1.TABLE_BUSINESS_MONTH, t1.REGION, t1.SEGMENT, t1.ACC_METH)
WITH DATA NO PRIMARY INDEX ON COMMIT PRESERVE ROWS;)
by teradata;
======================The full code below====================================
EXECUTE (create multiset volatile table CLC_&mon. AS
(SELECT
t1.TABLE_BUSINESS_MONTH,
t1.REGION,
Case when t2.SEGMENT is Null then 'NEW' else t2.SEGMENT end as "SEGMENT",
t1.ACC_METH,
COUNT(t1.ACC_N) as Count_ACC_N,
SUM(t1.NATIONAL_AMOUNT) AS NATIONAL_AMOUNT,
SUM(t1.NATIONAL_AMOUNT)/COUNT(t1.ACC_N) as 'ARPaU'n
FROM UAT_V_BASE.RA_FCLC_ACC t1
LEFT JOIN prd_sbx_general.&login._LIST_MK_2 t2 ON (t1.MARKET_CATEGORY=t2.MARKET_CATEGORY)
GROUP BY t1.TABLE_BUSINESS_MONTH, t1.REGION, "SEGMENT", t1.ACC_METH
WHERE SEGMENT in ('B2C','B2B','VIP','OTHER')
AND t1.ACC_METH in (1, 2)
AND TABLE_BUSINESS_MONTH between &firstmonth. and &lastmonth.
AND MRGN_ID=&MR.)
WITH DATA NO PRIMARY INDEX ON COMMIT PRESERVE ROWS;)
by teradata;
create table CLC_&mon. as select
TABLE_BUSINESS_MONTH FORMAT=Date9.,
REGION FORMAT=FSRGC.,
SEGMENT FORMAT=$9.,
ACC_METH FORMAT=F78C.,
Count_ACC_N,
NATIONAL_AMOUNT FORMAT=NUMX16.2 LABEL="(Сумма начислений с учетом налогов), RUB)",
'ARPaU'n
from connection to teradata
(select distinct
TABLE_BUSINESS_MONTH,
REGION,
SEGMENT,
ACC_METH,
Count_ACC_N,
NATIONAL_AMOUNT,
'ARPaU'n
GROUP BY TABLE_BUSINESS_MONTH, REGION, SEGMENT, ACC_METH, Count_ACC_N, NATIONAL_AMOUNT,'ARPaU'n
ORDER BY TABLE_BUSINESS_MONTH, REGION, SEGMENT, ACC_METH
from CLC_&mon.);
EXECUTE (create multiset volatile table Prev_Debt_&mon. AS
(SELECT
t1.TABLE_BUSINESS_MONTH,
t1.REGION,
Case when t2.SEGMENT is Null then 'NEW' else t2.SEGMENT end as "SEGMENT",
t1.ACC_METH,
t1.BASE_AGE,
Case when t1.BASE_AGE=-1 then 'ОДЗ'
when t1.BASE_AGE=3 then 'CДЗ'
else 'Другая ДЗ'
end as "Tип ДЗ",
Case when t1.NATIONAL_UNPAID between 0 and 200000 then '0...200'
when t1.NATIONAL_UNPAID between 200000 and 500000 then '200...500'
else '> 500' end as 'Amount_of_Debt'n,
t1.ACC_N,
SUM(t1.NATIONAL_UNPAID) AS NATIONAL_UNPAID,
Case when t1.EXCEPTED_FROM_BUH eq 0 then (t1.NT_BUH_DELTA + t1.NT_WRITE_OFF)
else t1.NT_WRITE_OFF_COLLFIN
end AS DEBT_EXPENSE
FROM UAT_V_BASE.RA_FDEB_ACC t1
LEFT JOIN prd_sbx_general.&login._LIST_MK_2 t2 ON (t1.MARKET_CATEGORY=t2.MARKET_CATEGORY)
WHERE SEGMENT in ('B2C','B2B','VIP','OTHER')
AND t1.ACC_METH in (1, 2)
AND TABLE_BUSINESS_MONTH between &firstmonth. and &lastmonth.
AND MRGN_ID=&MR.
GROUP BY t1.TABLE_BUSINESS_MONTH, t1.REGION, "SEGMENT", t1.ACC_METH, t1.BASE_AGE,'Tип ДЗ'n, t1.ACC_N, 'Amount_of_Debt'n, "DEBT_EXPENSE")
WITH DATA PRIMARY INDEX (ACC_N) ON COMMIT PRESERVE ROWS;)
by teradata;
EXECUTE (create multiset volatile table Debt_&mon. AS
(SELECT
t1.TABLE_BUSINESS_MONTH,
t1.REGION,
t1.SEGMENT,
t1.ACC_METH,
COUNT(t1.ACC_N) as Count_ACC_N,
SUM(t1.NATIONAL_UNPAID) AS NATIONAL_UNPAID,
SUM(t1.DEBT_EXPENSE) as DEBT_EXPENSE,
SUM(t2.NATIONAL_AMOUNT) as NATIONAL_AMOUNT,
(case when SUM(t2.NATIONAL_AMOUNT)=0 then 0 else (cast(SUM(t1.DEBT_EXPENSE) as Decimal(20,6))/cast(SUM(t2.NATIONAL_AMOUNT) as Decimal(20,6)) end) as 'РСДЗ/Д'n,
(case when SUM(t1.DEBT_EXPENSE)=0 then 0 else cast(SUM(t2.NATIONAL_AMOUNT) as Decimal(20,6))/cast(SUM(t1.DEBT_EXPENSE) as Decimal(20,6)) end) as 'Д/ДЗ'n
FROM Prev_Debt_&mon. t1
LEFT JOIN CLC_&mon. t2 on
t1.REGION=t2.REGION and
t1.SEGMENT=t2.SEGMENT and
t1.ACC_METH=t2.ACC_METH
GROUP BY t1.TABLE_BUSINESS_MONTH, t1.REGION, t1.SEGMENT, t1.ACC_METH)
WITH DATA NO PRIMARY INDEX ON COMMIT PRESERVE ROWS;)
by teradata;
Create table RSDZ_&mon. as select
TABLE_BUSINESS_MONTH FORMAT=Date9.,
REGION FORMAT=FSRGC.,
SEGMENT FORMAT=$9.,
ACC_METH FORMAT=F78C.,
Count_ACC_N,
NATIONAL_UNPAID FORMAT=NUMX16.2 ,
DEBT_EXPENSE FORMAT=NUMX16.2,
NATIONAL_AMOUNT FORMAT=NUMX16.2,
'РСДЗ/Д'n FORMAT=NUMX16.10,
'Д/ДЗ'n FORMAT=NUMX16.10
from connection to teradata
(select distinct
TABLE_BUSINESS_MONTH,
REGION,
SEGMENT,
ACC_METH,
Count_ACC_N,
NATIONAL_UNPAID,
DEBT_EXPENSE,
NATIONAL_AMOUNT,
'РСДЗ/Д'n,
'Д/ДЗ'n
GROUP BY TABLE_BUSINESS_MONTH, REGION, SEGMENT, ACC_METH, Count_ACC_N, NATIONAL_UNPAID, DEBT_EXPENSE, NATIONAL_AMOUNT, 'РСДЗ/Д'n, 'Д/ДЗ'n
ORDER BY TABLE_BUSINESS_MONTH, REGION, SEGMENT, ACC_METH
from Debt_&mon.);

Use Bigint instead of INTEGER?DECIMAL while casting your numeric columns

Actualy, I found how to solve my problem. I took my code where I had trouble apart from full code.
The log got the error "Numeric overflow occurred during computation." but sighned that the trouble is with column NATIONAL_AMOUNT.
Then I changed my SUM(t2.NATIONAL_AMOUNT) as NATIONAL_AMOUNT, on t2.NATIONAL_AMOUNT.
After that code had worked.
I agregated my t2.NATIONAL_AMOUNT on previous step and I should'n agregete it twice.

Related

Redshift Result size exceeds LISTAGG limit on svl_statementtext

Trying to reconstruct my query history from svl_statementtext using listagg.
Getting error :
Result size exceeds LISTAGG limit (limit: 65535)
However, I cannot see how or where I have exceeded limit.
My failing query :
SELECT pid,xid, min(starttime) AS starttime,
pg_catalog.listagg(
CASE WHEN (len(rtrim(("text")::text)) = 0) THEN ("text")::text ELSE rtrim(("text")::text) END
, ''::text
) WITHIN GROUP(ORDER BY "sequence")
AS query_statement
FROM svl_statementtext
GROUP BY pid,xid
HAVING min(starttime) >= '2022-06-27 10:00:00';
After the fail, I checked to see if I could find where the excessive size was coming from :
SELECT pid,xid, min(starttime) AS starttime,
SUM(OCTET_LENGTH(
CASE WHEN (len(rtrim(("text")::text)) = 0) THEN ("text")::text ELSE rtrim(("text")::text) END
)) as total_bytes
FROM svl_statementtext
GROUP BY pid,xid
HAVING min(starttime) >= '2022-06-27 10:00:00'
ORDER BY total_bytes desc;
However the largest size that this query reports is 2962
So how/why is listagg complaining about 65535 ??
Have seen some other posts mentioning using listaggdistinct, and catering for when the value being aggregated is null, but none seem to change my problem.
Any guidance appreciated :)
The longest string that Redshift can hold is 64K bytes. Listagg() is likely generating a string longer than this. The "text" column in svl_statementtext is 200 characters so if you have more than 319 segments you can overflow this string size.
The other issue I see is that your query will combine multiple statements into one string. You are only grouping by xid and pid which will give you all statements for a transaction. Add starttime to your group by list and this will break different statements into different results.
Also remember that xid and pid values repeat every few days so have some date range limit can help prevent a lot of confusion.
You need to add
where sequence < 320
to your query and also group by starttime.
Here's a query I have used to put together statements in Redshift:
select xid, pid, starttime, max(datediff('sec',starttime,endtime)) as runtime, type, listagg(regexp_replace(text,'\\\\n*',' ')) WITHIN GROUP (ORDER BY sequence) || ';' as querytext
from svl_statementtext
where pid = (SELECT pg_backend_pid()) --current session
and sequence < 320
and starttime > getdate() - interval '24 hours'
group by starttime, 1, 2, "type" order by starttime, 1 asc, "type" desc ;

How to reduce JSON object?

I have a JSON file created from a SQL query on a database. I'm trying to reduce several lines of the same "car_id" into a single line.
There is an example of my JSON file with several lines for a single :
[{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"1","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"2","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"3","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"4","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"5","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"6","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"}]
I searched in StackOverflow, Google, and either I did not understand how to reduce this file into this.
{"car_id":"1","sca_multiverseid":"430690","car_convertedmanacost":"2","car_coloridentity":"W","ccal_capacity":null,"clel_legality":"1,2,3,4,5,6","csul_supertype":null,"ctyl_type":"7","cstl_subtype":null,"set_block":"2","car_layout":"7","car_power":"0","car_toughness":"0"},
Here, there is only one different field to merge (clel_legality), but there can be several fields with different values (like (car_coloridentity, ccal_capacity, csul_supertype ...)
I'm sorry for my English if I made a mistake. Thank you in advance.
Edit :
There is my SQL Query :
SELECT car_id, sca_multiverseid, car_convertedmanacost, car_coloridentity, ccal_capacity, clel_legality, csul_supertype, ctyl_type, cstl_subtype, set_block, car_layout, car_power, car_toughness
FROM mag_card A LEFT JOIN mag_setcard B ON A.car_id = B.sca_card
LEFT JOIN mag_cardcapacityli C ON A.car_id=C.ccal_card
LEFT JOIN mag_cardlegalityli D ON A.car_id=D.clel_card
LEFT JOIN mag_cardsupertypeli E ON A.car_id=E.csul_card
LEFT JOIN mag_cardtypeli F ON A.car_id=F.ctyl_card
LEFT JOIN mag_cardsubtypeli G ON A.car_id=G.cstl_card
LEFT JOIN mag_set H ON B.sca_set=H.set_id
Thank's to Caleb McNevin :
SELECT car_id, sca_multiverseid, car_convertedmanacost, GROUP_CONCAT(DISTINCT car_coloridentity), GROUP_CONCAT(DISTINCT ccal_capacity), GROUP_CONCAT(DISTINCT clel_legality), GROUP_CONCAT(DISTINCT csul_supertype),GROUP_CONCAT(DISTINCT ctyl_type), GROUP_CONCAT(DISTINCT cstl_subtype), set_block, car_layout, car_power, car_toughness
FROM mag_card A LEFT JOIN mag_setcard B ON A.car_id = B.sca_card
LEFT JOIN mag_cardcapacityli C ON A.car_id=C.ccal_card
LEFT JOIN mag_cardlegalityli D ON A.car_id=D.clel_card
LEFT JOIN mag_cardsupertypeli E ON A.car_id=E.csul_card
LEFT JOIN mag_cardtypeli F ON A.car_id=F.ctyl_card
LEFT JOIN mag_cardsubtypeli G ON A.car_id=G.cstl_card
LEFT JOIN mag_set H ON B.sca_set=H.set_id
GROUP BY car_id
The command GROUP_CONCAT(DISTINCT xxx) combined a and a GROUP_BY(primary_key) at the end of the request has worked!

Oracle How do I transform this string field into structured data using regular expressions?

I did start at this answer:
Oracle 11g get all matched occurrences by a regular expression
But it didn't get me far enough. I have a string field that looks like this:
A=&token1&token2&token3,B=&token2&token3&token5
It could have any number of tokens and any number of keys. The desired output is a set of rows looking like this:
Key | Token
A | &token1
A | &token2
A | &token3
B | &token2
B | &token3
B | &token5
This is proving rather difficult to do.
I started here:
SELECT token from
(SELECT REGEXP_SUBSTR(str, '[A-Z=&]+', 1, LEVEL) AS token
FROM (SELECT 'A=&token1&token2&token3,B=&token2&token3&token5' str from dual)
CONNECT BY LEVEL <= LENGTH(REGEXP_REPLACE(str, '[A-Z=&]+', ',')))
Where token is not null
But that yields:
A=&
&
&
B=&
&
&
which is getting me nowhere. I'm thinking I need to do a nested clever select where the first one gets me
A=&token1&token2&token3
B=&token2&token3&token5
And a subsequent select might be able to do a clever extract to get the final result.
Stumped. I'm trying to do this without using procedural or function code -- I would like the set to be something I can union with other queries so if it's possible to do this with nested selects that would be great.
UPDATE:
SET DEFINE OFF
SELECT SUBSTR(token,1,1) as Key, REGEXP_SUBSTR(token, '&\w+', 1, LEVEL) AS token2
FROM
(
-- 1 row per key/value pair
SELECT token from
(SELECT REGEXP_SUBSTR(str, '[^,]+', 1, LEVEL) AS token
FROM (SELECT 'A=&token1&token2&token3,B=&token2&token3&token5' str from dual)
CONNECT BY LEVEL <= LENGTH(REGEXP_REPLACE(str, '[^,]+', ',')))
Where token is not null
)
CONNECT BY LEVEL <= LENGTH(REGEXP_REPLACE(token, '&\w+'))
This gets me
A | &token1
A | &token2
B | &token3
B | &token2
A | &token2
B | &token3
Which is fantastic formatting except for the small problem that it's wrong (A should have a token3, and token4 and token5 are nowhere to be seen).
Great question! Thanks for it!
select distinct k, regexp_substr(v, '[^&]+', 1, level) t
from (
select substr(regexp_substr(val,'^[^=]+=&'),1,length(regexp_substr(val,'^[^=]+=&'))-2) k, substr(regexp_substr(val,'=&.*'),3) v
from (
select regexp_substr(str, '[^,]+', 1, level) val
from (select 'A=&token1&token2&token3,B=&token2&token3&token5' str from dual)
connect by level <= length(str) - length(replace(str,','))+1
)
) connect by level <= length(v) - length(replace(v,'&'))+1
It is an answer, and one that seems to work... But I don't like the middle splitting the val into kand v- there must be a better way (if the Key is always one character, that makes it easy though) . And having to put a DISTINCT to get rid of duplicates is horrible... Maybe with further playing you can clean it up though (or someone else might)
EDIT based on keeping the leading & and the key being a single character:
select distinct k, regexp_substr(v, '&[^&]+', 1, level) t
from (
select substr(val,1,1) k
, substr(regexp_substr(val,'=&.*'),1) v
from (
select regexp_substr(str, '[^,]+', 1, level) val
from (select 'A=&token1&token2&token3,B=&token2&token3&token5' str from dual)
connect by level <= length(str) - length(replace(str,','))+1
)
) connect by level < length(v) - length(replace(v,'&'))+1

Getting last rows by date on a large postgresl data base

I have a large database ( there are 6000 rows inserted per minute ) on a partitioned table, I was working cool when the database was small, now I have a large database. I was using this solution a previous solutions SQL joined by date but it is using 250MB of hard disk and grows up while my table grows, then I decide on change it to an iteration of simple queries, that works well for 10 rows, but get slow with more than 10 cars ( 15 secods for response ) and uses more than 200MB of hard disk.
My question is how to build a good query faster for resolve this issue
Extra info
Query is called on a django app by ajax
I was thinking on iterative ajax calls instead one call with full items list response
Tables are partitioned by day
My actual query is
CREATE OR REPLACE FUNCTION gps_get_last_positions (
_plates varchar(8)
)
RETURNS TABLE (
plate varchar,
device_id integer,
date_time_process timestamp with time zone,
latitude double precision,
longitude double precision,
course smallint,
speed smallint,
mileage integer,
gps_signal smallint,
gsm_signal smallint,
alarm_status boolean,
gsm_status boolean,
vehicle_status boolean,
alarm_over_speed boolean,
other text,
realtime double precision
) AS $func$
DECLARE
arr varchar[];
BEGIN
arr := regexp_split_to_array(_plates, E'\\s+');
FOR i IN 1..array_length(arr, 1) LOOP
RETURN QUERY SELECT
gpstracking_vehicles.registration,
gpstracking_device_tracks.device_id,
gpstracking_device_tracks.date_time_process,
gpstracking_device_tracks.latitude,
gpstracking_device_tracks.longitude,
gpstracking_device_tracks.course,
gpstracking_device_tracks.speed,
gpstracking_device_tracks.mileage,
gpstracking_device_tracks.gps_signal,
gpstracking_device_tracks.gsm_signal,
gpstracking_device_tracks.alarm_status,
gpstracking_device_tracks.gps_status,
gpstracking_device_tracks.vehicle_status,
gpstracking_device_tracks.alarm_over_speed,
gpstracking_device_tracks.other,
EXTRACT(EPOCH FROM current_timestamp - gpstracking_device_tracks.date_time_process)/60 AS realtime
FROM (
gpstracking_devices INNER JOIN (
gpstracking_vehicles INNER JOIN gpstracking_vehicles_devices ON ( gpstracking_vehicles.id = gpstracking_vehicles_devices.vehicle_id AND gpstracking_vehicles_devices.is_joined = TRUE )
) ON ( gpstracking_devices.id = gpstracking_vehicles_devices.device_id AND gpstracking_vehicles_devices.is_joined = TRUE )
) INNER JOIN gpstracking_device_tracks ON ( gpstracking_devices.id = gpstracking_device_tracks.device_id )
WHERE gpstracking_vehicles.registration = arr[i]::VARCHAR
ORDER BY gpstracking_device_tracks.date_time_process DESC
LIMIT 1;
END LOOP;
RETURN;
END;
$func$
LANGUAGE plpgsql VOLATILE SECURITY DEFINER;
Configuration params
application_name phpPgAdmin_5.0.4 client
constraint_exclusion on configuration file
DateStyle ISO, MDY session
default_text_search_config pg_catalog.english configuration file
external_pid_file /var/run/postgresql/9.1-main.pid configuration file
lc_messages en_US.UTF-8 configuration file
lc_monetary en_US.UTF-8 configuration file
lc_numeric en_US.UTF-8 configuration file
lc_time en_US.UTF-8 configuration file
log_line_prefix %t configuration file
log_timezone localtime environment variable
max_connections 100 configuration file
max_stack_depth 2MB environment variable
port 5432 configuration file
shared_buffers 24MB configuration file
ssl on configuration file
TimeZone localtime environment variable
unix_socket_directory /var/run/postgresql configuration file
My first slow query was:
CREATE OR REPLACE VIEW view_vehicle_devices AS
SELECT
gpstracking_vehicles_devices.id AS id,
gpstracking_devices.id AS device_id,
gpstracking_vehicles.id AS vehicle_id,
gpstracking_drivers.id AS driver_id,
gpstracking_device_protocols.name AS protocol,
gpstracking_vehicles.registration AS plate,
gpstracking_drivers.firstname as first_name,
gpstracking_drivers.lastname as last_name,
gpstracking_devices.imei,
gpstracking_devices.simcard,
gpstracking_device_tracks.date_time_process,
gpstracking_device_tracks.latitude,
gpstracking_device_tracks.longitude,
gpstracking_device_tracks.course,
gpstracking_device_tracks.speed,
gpstracking_device_tracks.mileage,
gpstracking_device_tracks.gps_signal,
gpstracking_device_tracks.gsm_signal,
gpstracking_device_tracks.alarm_status,
gpstracking_device_tracks.gps_status,
gpstracking_device_tracks.vehicle_status,
gpstracking_device_tracks.alarm_over_speed,
gpstracking_device_tracks.other,
gpstracking_device_tracks.point,
EXTRACT(EPOCH FROM current_timestamp - gpstracking_device_tracks.date_time_process)/60 realtime,
gpstracking_devices.created,
gpstracking_devices.updated,
gpstracking_devices.is_connected
FROM (
gpstracking_vehicles LEFT JOIN (
gpstracking_drivers LEFT JOIN gpstracking_vehicles_drivers ON gpstracking_drivers.id = gpstracking_vehicles_drivers.driver_id AND gpstracking_vehicles_drivers.is_joined = TRUE
) ON gpstracking_vehicles.id = gpstracking_vehicles_drivers.vehicle_id AND gpstracking_vehicles_drivers.is_joined = TRUE
) LEFT JOIN (((
gpstracking_device_protocols RIGHT JOIN gpstracking_devices ON gpstracking_device_protocols.id = gpstracking_devices.device_protocol_id
) LEFT JOIN (
SELECT DISTINCT ON (gpstracking_device_tracks.device_id) gpstracking_device_tracks.device_id,
gpstracking_device_tracks.date_time_process,
gpstracking_device_tracks.latitude,
gpstracking_device_tracks.longitude,
gpstracking_device_tracks.course,
gpstracking_device_tracks.speed,
gpstracking_device_tracks.mileage,
gpstracking_device_tracks.gps_signal,
gpstracking_device_tracks.gsm_signal,
gpstracking_device_tracks.alarm_status,
gpstracking_device_tracks.gps_status,
gpstracking_device_tracks.vehicle_status,
gpstracking_device_tracks.alarm_over_speed,
gpstracking_device_tracks.other,
gpstracking_device_tracks.point
FROM gpstracking_device_tracks
ORDER BY gpstracking_device_tracks.device_id, gpstracking_device_tracks.date_time_process DESC
) AS gpstracking_device_tracks ON gpstracking_devices.id = gpstracking_device_tracks.device_id
) LEFT JOIN gpstracking_vehicles_devices ON ( gpstracking_devices.id = gpstracking_vehicles_devices.device_id AND gpstracking_vehicles_devices.is_joined = TRUE )
) ON ( gpstracking_vehicles.id = gpstracking_vehicles_devices.vehicle_id AND gpstracking_vehicles_devices.is_joined = TRUE )
I have changed it for the loop that is starting my post, my loop rujns faster, however is not enought faster as I need
Your problem is that a planner can not know in which partition is an answer to your query. It only has statistics. So you do not benefit from partitioning your data by a day at all.
To benefit from it you can modify your query so it'll look for latest coordinates from current day, if not found then from yesterday, if not found from a day before an so on. I suppose 99% answers would be found in todays partition only.
Or you can partition by for example device_id % 256 instead.
But even better would be to create an additional table with several recent device coordinates only. It would be maintained with a trigger on gpstracking_device_tracks which will just do (pseudocode):
if random()*64 < 1.0 then
-- statistically once per 64 runs do a cleanup
with todelete as (
-- lock rows in particular order to avoid possible deadlocks
-- if run concurrently
select id from gpstracking_device_tracks_recent where device_id=?
order by id for share;
)
delete from gpstracking_device_tracks_recent
where id in (select id from todelete)
end if;
insert into gpstracking_device_tracks_recent (...) values (...);
And then look for latest coordinates in this much smaller table.

SQL Comparison to a value in the next row

I have been a long time reader of this forum, it has helped me a lot, however I have a question which I cant find a solution specific to my requirements, so this is the first time I have had to ask anything.
I have a select statement which returns meter readings sorted by date (the newest readings at the top), in 99.9% of cases the meter readings always go up as the date moves on, however due to system errors occasionally some go down, I need to identify instances where the reading in the row below (previous reading) is GREATER than the latest reading (Current cell)
I have come across the LEAD function, however its only in Oracle or SS-MS-2012, I'm using SS-MS-2008.
Here is a simplified version of my select statment:
SELECT Devices.SerialNumber,
MeterReadings.ScanDateTime,
MeterReadings.TotalMono,
MeterReadings.TotalColour
FROM dbo.MeterReadings AS MeterReadings
JOIN DBO.Devices AS Devices
ON MeterReadings.DeviceID = Devices.DeviceID
WHERE Devices.serialnumber = 'ANY GIVEN DEVICE SERIAL NUMBER'
AND Meterreadings.Scandatetime > 'ANY GIVEN SCAN DATE TIME'
ORDER BY MeterReadings.ScanDateTime DESC, Devices.SerialNumber ASC
This is the code I used in the end
WITH readings AS
(
SELECT
d.SerialNumber
, m.TotalMono
, m.TotalColour
, m.ScanDateTime
FROM dbo.MeterReadings m
INNER JOIN dbo.Devices d ON m.DeviceId = d.DeviceId
WHERE m.ScanDateTime > '2012-01-01'
)
SELECT top 1 *
FROM readings r
LEFT JOIN readings p ON p.SerialNumber = r.SerialNumber
and p.ScanDateTime < r.ScanDateTime
and p.TotalMono > r.TotalMono
order by r.serialnumber, p.TotalMono desc, r.TotalMono asc
Try something like this.
;WITH readings AS
(
SELECT
d.SerialNumber
, m.TotalMono
, m.TotalColour
, m.ScanDateTime
FROM dbo.MeterReadings m
INNER JOIN dbo.Devices d ON m.DeviceId = d.DeviceId
)
SELECT *
FROM readings r
LEFT JOIN readings p ON p.SerialNumber = r.SerialNumber
AND p.ScanDateTime < r.ScanDateTime
WHERE p.reading > r.reading