Open njouanin opened 23 hours ago
Problème vient des données brutes (spoofing) -> modifier le traitement pour ignorer les segments qui ont des timestamp_end < à timestamp_start.
select * from fct_segment fs where fs.segment_duration < '00:00';
id | excursion_id | timestamp_start | timestamp_end | segment_duration | course | distance | average_speed | type | in_amp_zone | in_territorial_waters | in_zone_with_no_fishing_rights | last_vessel_segment | created_at | updated_at | start_position | end_position | speed_at_start | speed_at_end | heading_at_start | heading_at_end ---------+--------------+------------------------+------------------------+--------------------+--------+--------------------+------------------------+--------+-------------+-----------------------+--------------------------------+---------------------+-------------------------------+-------------------------------+----------------------------------------------------+----------------------------------------------------+---------------------+---------------------+------------------+---------------- 5196702 | 70590 | 2024-09-01 14:26:33+02 | 2024-08-26 04:35:50+02 | -7 days +14:09:17 | | 31.675765537788706 | -0.20589364844556915 | AT_SEA | f | f | f | f | 2024-09-07 18:51:00.162412+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E61000000E3DB53C07A915403A1305C019974A40 | 0101000020E6100000475A2A6F47381340B1506B9A77BA4A40 | 2.7958060850573045 | 1.9446151671543859 | 113 | 319 5488068 | 71121 | 2024-09-07 18:02:17+02 | 2024-07-19 10:15:22+02 | -51 days +16:13:05 | | 7.3859139271481675 | -0.0061152710231527265 | AT_SEA | f | f | f | f | 2024-09-11 18:11:42.446872+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000F6285C8FC2F5F63F690D1092A6744340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 7.93501128798678 | 0.39574310646259453 | 171 | 91 4895605 | 66258 | 2024-08-26 18:15:02+02 | 2024-07-19 10:15:22+02 | -39 days +16:00:20 | | 7.3859139271481675 | -0.008028215791681533 | AT_SEA | f | f | f | f | 2024-09-01 17:28:21.05848+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000F6285C8FC2F5F63F690D1092A6744340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 148.79637580168986 | 0.40099724524155816 | 170 | 91 5232777 | 71087 | 2024-09-01 16:41:48+02 | 2024-07-19 10:15:22+02 | -45 days +17:33:34 | | 10.387656087110994 | -0.009777164503739444 | AT_SEA | f | f | f | f | 2024-09-07 18:51:00.162412+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E61000009E6469AF57ABF63F96888594FE5D4340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 0.4741995583210178 | 0.5140610410268184 | 265 | 91 6296180 | 86905 | 2024-09-25 17:45:00+02 | 2024-07-19 10:15:22+02 | -69 days +16:30:22 | | 6.332326192107835 | -0.003862366450416813 | AT_SEA | f | f | f | f | 2024-10-06 10:53:37.140721+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000795EE8191420F73FF168E388B5724340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 0.4449178160364368 | 0.33336992248803504 | 354 | 91 7038668 | 94429 | 2024-10-24 16:28:54+02 | 2024-10-21 07:05:13+02 | -4 days +14:36:19 | | 26.41899643821961 | -0.3245787406963685 | AT_SEA | f | f | f | f | 2024-10-28 21:57:18.703216+01 | 2024-11-05 21:42:56.308364+01 | 0101000020E610000051DA1B7C61F207409F71E14048B64940 | 0101000020E61000001A888641700F0340E9E136E094B44940 | 0.9865448438149889 | 1.57186192841009 | 91 | 259 6854181 | 94429 | 2024-10-19 17:41:21+02 | 2024-10-18 07:55:11+02 | -2 days +14:13:50 | | 11.549792814719526 | -0.34201903539516565 | AT_SEA | f | f | f | f | 2024-10-24 16:35:44.680284+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000621378ABD7BA0C405EC5779C01B94940 | 0101000020E61000009199EA4D35A00A4021BD137933B54940 | 4.311001246853876 | 0.7052778404504593 | 192 | 82 6040055 | 80253 | 2024-09-18 15:40:31+02 | 2024-09-18 15:16:32+02 | -1 days +23:36:01 | | 0.9183485017208955 | -2.2974667173003644 | AT_SEA | f | f | f | f | 2024-09-25 17:59:58.055445+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000BA83D89942A7FD3FE2BAF7AAF4AC4940 | 0101000020E6100000E73BB66D3152FD3FE6DDC3EB9BAC4940 | 188.0319405306178 | 0.03381413694310804 | 51 | 84 6420863 | 89024 | 2024-10-02 17:30:05+02 | 2024-10-01 11:00:44+02 | -2 days +17:30:39 | | 733.4996119489387 | -24.05771269409152 | AT_SEA | f | f | f | f | 2024-10-12 12:42:37.250145+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000C667F49B709CFB3FC7A80052FAAB4940 | 0101000020E6100000A5A61B306187FA3FC207E1979B5C4440 | 0.2989807189815098 | 36.39996566778579 | 352 | 178 6510228 | 89024 | 2024-10-09 17:14:32+02 | 2024-10-03 02:20:29+02 | -7 days +09:05:57 | | 103.2185346199172 | -0.6495783090286952 | AT_SEA | f | f | f | f | 2024-10-12 17:57:42.827941+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E610000017D6AE8530F10340A2FBA4A07C5E4A40 | 0101000020E6100000B7D100DE02C9FB3FF425E9D4F4A84940 | 3.7266954022109005 | 9.857436073024607 | 176 | 17 5618725 | 74687 | 2024-09-11 14:24:04+02 | 2024-09-07 06:30:20+02 | -5 days +16:06:16 | | 4031.077770473164 | -38.79932831503697 | AT_SEA | f | f | f | f | 2024-09-18 15:50:51.419634+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000371BE914FD34FB3FACD75F3C24AB4B40 | 0101000020E6100000D40E7F4DD6B75BC0325A475513984A40 | 7.759088968268934 | 217.5120539836309 | 153 | 98 6251644 | 86135 | 2024-09-25 17:40:39+02 | 2024-09-25 01:51:37+02 | -1 days +08:10:58 | | 56.35214329969732 | -3.5627079463122184 | AT_SEA | f | f | f | f | 2024-10-06 10:53:37.140721+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E61000003ED00A0C591DFD3F90662C9ACE9E4940 | 0101000020E6100000CD237F30F0BC0840513CFC6F84B44940 | 3.3763449841091173 | 5.984358212342511 | 128 | 281 5464917 | 74686 | 2024-09-07 14:40:39+02 | 2024-09-07 06:30:20+02 | -1 days +15:49:41 | | 4241.740924993643 | -519.0614001147936 | AT_SEA | f | f | f | f | 2024-09-11 18:11:42.446872+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000489ACB4FA2A81540D75E7615F3964A40 | 0101000020E6100000D40E7F4DD6B75BC0325A475513984A40 | 0.30747824118186695 | 232.875860941313 | 343 | 98 7143435 | 94429 | 2024-10-28 21:09:49+01 | 2024-10-28 06:24:08+01 | -1 days +09:14:19 | | 55.039399822720966 | -3.7286057726011084 | AT_SEA | f | f | f | f | 2024-10-30 13:45:25.298199+01 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000B3F0B3995CDFF53F752FC58F907A4940 | 0101000020E6100000514B94396DA003407417F6EEEEAE4940 | 11.876734791824415 | 5.176961118191749 | 223 | NaN 6631016 | 94428 | 2024-10-12 01:10:28+02 | 2024-10-03 02:20:29+02 | -9 days +01:10:01 | | 144.9663586393712 | -0.6747860950191769 | AT_SEA | f | f | f | f | 2024-10-24 09:57:16.025093+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E61000008A592F8672521240898DE4B8F43A4A40 | 0101000020E6100000B7D100DE02C9FB3FF425E9D4F4A84940 | 9.85584379657216 | 107.95057239096933 | 330 | 17 7171303 | 94429 | 2024-10-29 21:09:07+01 | 2024-10-28 06:24:08+01 | -2 days +09:15:01 | | 92.45845672624203 | -2.3860417939517222 | AT_SEA | f | f | f | f | 2024-11-01 21:33:56.899399+01 | 2024-11-05 21:42:56.308364+01 | 0101000020E610000093E5A05B1AE5EB3FFBDB33852D3D4940 | 0101000020E6100000514B94396DA003407417F6EEEEAE4940 | 1.4359804822879871 | 8.685598819174265 | 273 | NaN 6445818 | 89622 | 2024-10-02 17:25:54+02 | 2024-07-19 10:15:22+02 | -76 days +16:49:28 | | 9.01421086635153 | -0.004988010621679979 | AT_SEA | f | f | f | f | 2024-10-12 12:42:37.250145+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000C520B0726891F63FAA038B896E624340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 0.32918188282989724 | 0.46558060553355335 | 352 | 91 6093155 | 86836 | 2024-09-18 15:26:06+02 | 2024-07-19 10:15:22+02 | -62 days +18:49:16 | | 7.3859139271481675 | -0.005027239353224024 | AT_SEA | f | f | f | f | 2024-09-25 17:59:58.055445+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000F6285C8FC2F5F63F690D1092A6744340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 0 | 0.34100981862593105 | 170 | 91 6526880 | 91815 | 2024-10-09 17:30:12+02 | 2024-07-19 10:15:22+02 | -83 days +16:45:10 | | 8.917054804934537 | -0.004514399364603352 | AT_SEA | f | f | f | f | 2024-10-12 17:57:42.827941+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000BA490C022B87F63F9B5AB6D617634340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 4.620982650232351 | 0.4625327707683737 | 1 | 91 6678723 | 93084 | 2024-10-12 17:45:10+02 | 2024-07-19 10:15:22+02 | -86 days +16:30:12 | | 6.672521953404185 | -0.0032588682863484604 | AT_SEA | f | f | f | f | 2024-10-24 09:57:16.025093+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E610000043002ED0071EF73F8BB1E123C1734340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 32.036034759015244 | 0.35133890207795526 | 359 | 91 5667347 | 75031 | 2024-09-11 18:00:01+02 | 2024-07-19 10:15:22+02 | -55 days +16:15:21 | | 7.3859139271481675 | -0.005665155876426293 | AT_SEA | f | f | f | f | 2024-09-18 15:50:51.419634+02 | 2024-11-05 21:42:56.308364+01 | 0101000020E6100000F6285C8FC2F5F63F690D1092A6744340 | 0101000020E610000083C0CAA145B6F83F23DBF97E6A6C4340 | 0 | 0.3948234184449785 | 170 | 91
A voir si le problème vient de la qualité des données SPIRE ou d'un "bug" de l'ETL.
Je fouille pour déceler l'origine
Premier cas : mmsi utlisé pendant 1 court moment par un autre navire => problème dans les données brutes 246557000_ais_data_subset.csv
Problème vient des données brutes (spoofing) -> modifier le traitement pour ignorer les segments qui ont des timestamp_end < à timestamp_start.
select * from fct_segment fs where fs.segment_duration < '00:00';
A voir si le problème vient de la qualité des données SPIRE ou d'un "bug" de l'ETL.