search

joshuaulrich / xts

Extensible time series class that provides uniform handling of many R time series classes by extending zoo.
http://joshuaulrich.github.io/xts/
GNU General Public License v2.0
219 stars 70 forks source link

xts dates subsetting issues #400

Closed SamoPP closed 1 year ago

SamoPP commented 1 year ago

Description

While subsetting xts object by dates (series$IsNthTradingDayOfMonth[lastNthTradingDaysOfMonth]) nothing is returned.

Subestting using series$IsNthTradingDayOfMonth[which(index(series) %in% lastNthTradingDaysOfMonth)] works as expected.

See code below for details.

What is it that I do not understand?

Expected behavior

I was assuming the subsetting using vector of dates would work.

Minimal, reproducible example

> dput(series)
structure(c(146.41, 143.61, 144.13, 142.6, 143.13, 141.65, 142.88, 
143.9, 143.39, 143.42, 140.96, 140.53, 141.37, 143.4, 144.3, 
143.11, 142.26, 142.41, 142.09, 142.18, 142.28, 141.95, 140.44, 
139.85, 138.91, 138.21, 138.72, 137.66, 136.85, 137.13, 135.6, 
135.95, 136.45, 137.52, 137.69, 137.68, 139.09, 136.86, 138.43, 
140.36, 139.84, 137.49, 140.36, 139.12, 137.51, 137.06, 134.83, 
134.34, 132.7, 132.86, 131.83, 132.82, 132.57, 131.28, 128.95, 
129.37, 129.65, 129.5, 129.39, 130.56, 130.07, 131.98, 130.01, 
131.93, 130.76, 126.43, 126.11, 125.45, 123.92, 124.05, 123.05, 
122.86, 120.87, 119.15, 120.22, 120.83, 120.07, 121.38, 122.44, 
122.19, 120.39, 119.31, 117.86, 119.04, 117.97, 116.35, 114.13, 
112.86, 115.89, 114.74, 118.02, 116.98, 116.21, 114.71, 114.7, 
118.64, 117.09, 117.71, 117.9, 119.61, 119.02, 119.36, 117.18, 
117.03, 116.63, 115.08, 115.15, 114.61, 114.68, 113.84, 114.36, 
111.49, 110.53, 110.09, 108.45, 112.01, 110.08, 112.69, 113.9, 
113.45, 111.22, 111.29, 112.38, 114.84, 115.9, 116.89, 117.39, 
115.1, 113.5, 113.79, 115.51, 113.77, 115.08, 115.84, 115.25, 
114.91, 115.48, 115.19, 118.29, 116.71, 118.86, 118.01, 118.06, 
117.7, 118.22, 120.15, 117.17, 118.98, 117.07, 117.64, 117.73, 
117.73, 116.86, 115.1, 116.33, 115.25, 115.06, 114.96, 113.23, 
112.86, 112.19, 111.48, 111.32, 112.3, 112.94, 112.62, 112.69, 
109.87, 109.68, 108.89, 108.32, 108.86, 108.33, 108.63, 106.56, 
107.34, 107.85, 107.03, 106.95, 105.69, 106.7, 105.79, 105.35, 
105.35, 102.71, 103.13, 102.9, 104.27, 103.85, 104.15, 102.73, 
102.6, 100.81, 100.5, 99.74, 99.28, 98.36, 100.19, 99.29, 97.95, 
97.19, 95.93, 92.99, 92.82, 94.57, 95.77, 96.98, 96.86, 96.34, 
97.38, 96.91, 95.17, 95.1, 94.77, 93.8, 93.82, 96.73, 97.57, 
97.91, 98.25, 99.99, 100.21, 100.6, 100.69, 100.65, 102.14, 102.61, 
103.61, 102.15, 101.62, 103.17, 105.3, 105.85, 106.5, 108.37, 
108.7, 107.94, 107.86, 109.04, 107.66, 108.39, 106.6, 105.83, 
103.38, 104.29, 103.67, 102.58, 100.49, 100.65, 99.86, 99.87, 
102.2, 103.22, 101.87, 102.7, 104.41, 104.53, 104.97, 105.82, 
107.1, 105.67, 108.46, 108.1, 106.96, 105.57, 106.12, 107.37, 
107.14, 106.19, 106.46, 106.98, 107.55, 108.92, 106.77, 105.75, 
105.33, 105.04, 106.47, 104.4, 103.71, 104.2, 103.55, 102.04, 
101.1, 101, 100.99, 101.54, 101.44, 101.29, 100.57, 100.82, 99.06, 
100.93, 102.22, 101.46, 102.4, 101.55, 104.02, 108.2, 105.14, 
106.78, 107.38, 106.65, 106.88, 105.08, 104.66, 105.36, 107.09, 
105.05, 104.24, 103.7, 104.46, 105.35, 105.88, 105.78, 107.77, 
108.42, 107.64, 107.05, 107.05, 107.09, 105.39, 104.42, 103.81, 
103.91, 104.83, 105.3, 104.96, 106.34, 106.79, 105.19, 106.21, 
105.27, 104, 106.02, 105.22, 104.34, 103.52, 103.5, 103.67, 105.3, 
105.04, 103.39, 102.4, 103.08, 102.2, 101.26, 101.25, 100.43, 
101.29, 100.7, 100.12, 101.4, 101.99, 103.28, 103, 101.35, 101.84, 
102.17, 100.87, 101.6, 102.24, 102.11, 101.63, 103.03, 102.44, 
146.94, 144.13, 144.16, 143.44, 143.24, 142.81, 143.63, 144.01, 
144.42, 143.46, 141.31, 141.71, 141.99, 144.1, 144.37, 143.71, 
142.57, 143.37, 143.46, 142.95, 142.36, 143.57, 141.48, 140.14, 
139.31, 138.58, 139.37, 137.83, 138.51, 137.74, 135.84, 136.07, 
137.57, 138.37, 138.67, 137.85, 139.37, 137.31, 140.05, 142.33, 
140.42, 138.7, 140.83, 140.64, 138.25, 137.35, 135.17, 135.45, 
132.89, 133.15, 133.13, 133.34, 133.54, 131.67, 129.34, 131.6, 
131.18, 129.52, 130.56, 131.55, 132.01, 132.67, 132.96, 131.98, 
130.78, 128.3, 126.67, 125.83, 124.19, 124.16, 124.3, 122.93, 
121.08, 119.72, 121.97, 120.91, 121.44, 122.19, 123.03, 122.32, 
121.07, 120.97, 118.27, 119.31, 118.97, 116.47, 115.04, 114.71, 
116.96, 117.94, 118.89, 117.15, 116.94, 115.35, 117, 118.72, 
119.02, 118.25, 119.64, 119.62, 119.09, 119.74, 117.23, 117.34, 
116.71, 116.06, 115.51, 115.78, 115.13, 114.71, 114.61, 111.67, 
110.98, 110.99, 111.72, 112.83, 111.02, 113.31, 115.17, 114.17, 
112.22, 112.18, 113.9, 115.77, 117.59, 117.75, 117.54, 115.17, 
113.5, 114.79, 116.23, 116.65, 116.16, 116.82, 115.32, 115.15, 
115.55, 116.72, 119.27, 117.54, 119.24, 118.49, 118.92, 119.31, 
120.1, 120.69, 119.46, 119.55, 117.11, 118.53, 118.36, 118.91, 
117.06, 115.57, 116.75, 116.02, 115.1, 115.64, 113.34, 112.96, 
113.48, 111.87, 113.09, 113.99, 113.01, 113.51, 113.34, 110.25, 
110.56, 108.97, 109.48, 109.52, 108.8, 108.89, 107.74, 108.21, 
108.19, 108.05, 107.82, 106.84, 108.04, 105.96, 106.43, 105.41, 
102.98, 104.45, 104.23, 104.86, 104.68, 104.65, 102.84, 102.84, 
101.62, 100.51, 100.9, 100.54, 100.38, 100.19, 99.66, 98.46, 
97.42, 96.33, 93.93, 93.55, 95.23, 96.69, 97.9, 97.63, 96.62, 
97.42, 97.63, 96.38, 95.66, 94.86, 94.96, 95.02, 98.43, 98.37, 
97.99, 99.3, 101.45, 100.57, 100.89, 100.82, 101.62, 103.25, 
102.93, 103.81, 102.96, 102.73, 105.81, 107.1, 105.99, 107.39, 
109.68, 109.45, 108.17, 108.11, 109.08, 108.32, 109.12, 107.83, 
105.84, 103.81, 104.4, 104.12, 102.81, 101.07, 100.78, 100.99, 
100.55, 102.34, 103.48, 103.29, 105.3, 105.93, 104.82, 105.72, 
107.81, 107.8, 106.66, 108.77, 108.37, 107.24, 106.2, 107.45, 
107.91, 107.65, 106.96, 107.16, 107.28, 108.59, 109.36, 107.11, 
106.31, 106.23, 105.58, 106.62, 104.44, 104.42, 104.83, 103.95, 
102.22, 102.38, 101.2, 101.69, 102.59, 101.8, 101.7, 101.72, 
101.03, 99.65, 101.9, 102.34, 102.37, 102.97, 102.52, 105.7, 
109.1, 105.61, 107.71, 108.03, 107.73, 106.96, 105.42, 106.45, 
106.55, 107.43, 105.76, 104.68, 104.45, 105.05, 106.56, 107.1, 
107.8, 108.37, 108.87, 107.77, 107.09, 107.17, 107.39, 105.55, 
104.59, 104.59, 104.24, 105.28, 105.46, 105.49, 107.13, 107.24, 
105.27, 106.76, 105.42, 105.7, 106.6, 106.45, 104.9, 103.87, 
103.64, 104.24, 105.52, 105.25, 103.47, 102.89, 103.16, 102.26, 
101.86, 101.71, 101.25, 101.36, 100.8, 101.18, 102.24, 103.18, 
103.63, 103.07, 102.44, 102.47, 102.43, 102.14, 102.27, 102.3, 
102.33, 102.29, 103.63, 102.83, 144.28, 142.74, 142.71, 142.29, 
141.56, 141.36, 142.69, 142.99, 143.03, 141.79, 140.02, 140.31, 
141.09, 142.67, 142.43, 141.8, 140.49, 142.33, 141.85, 141.96, 
141.05, 141.93, 140.18, 138.78, 138.55, 137.82, 138.19, 135.84, 
135.36, 135.97, 134.94, 134.51, 135.9, 137.16, 137.47, 136.56, 
136.25, 136.18, 138.32, 140, 136.41, 137.02, 139.3, 138.77, 137.25, 
136.13, 133.72, 134.21, 131.72, 131.19, 130.32, 131.07, 132.57, 
129.75, 128.34, 128.86, 129.6, 127.65, 129.04, 129.74, 129.98, 
131.66, 129.71, 130.71, 128.29, 126.4, 125.41, 124.26, 122.56, 
122.67, 122.89, 120.38, 119.69, 118.67, 119.92, 119.17, 119.81, 
121.12, 121.76, 120.77, 120.01, 119.1, 116.93, 117.89, 117.35, 
114.01, 113.31, 112.62, 115.48, 114.44, 117.65, 115.9, 115.85, 
114.36, 114.61, 116.85, 117.06, 116.54, 117.86, 118.59, 117.8, 
118.74, 115.92, 115.69, 115.46, 114.87, 113.81, 114.58, 114.08, 
113.68, 112.96, 109.24, 108.59, 109.03, 108.11, 110.96, 109.81, 
112.38, 113.49, 112.5, 111.16, 110.87, 112.17, 114.75, 115.22, 
116.17, 114.77, 113.51, 112.05, 113.7, 114.97, 113.45, 114.47, 
115.6, 114.18, 113.66, 114.05, 115.01, 117.97, 116.47, 117.39, 
116.71, 117.32, 117.33, 118.08, 117.28, 116.33, 118.54, 115.89, 
117.56, 117.46, 116.73, 114.09, 114.43, 115.45, 114.24, 114.31, 
114.58, 112.74, 112.13, 111.73, 110.86, 110.99, 112.01, 112.13, 
112.21, 111.57, 108.94, 109.47, 107.42, 108.21, 108.06, 107.63, 
106.92, 106.24, 107.11, 107.58, 106.76, 106.8, 105.4, 105.97, 
104.73, 104.56, 103.22, 100.9, 102.48, 102.8, 102.22, 103.15, 
103.3, 101.58, 101.53, 100.47, 98.79, 98.99, 99.19, 98.33, 98.25, 
97.94, 96.87, 96.32, 94.74, 92.26, 91.85, 94.44, 95.76, 96.21, 
96.41, 95.03, 96.21, 95.96, 95.09, 94.22, 93.24, 93.76, 93.69, 
96.64, 97.54, 97.09, 98.11, 99.74, 99.78, 99.54, 99.8, 100.53, 
102.08, 102.43, 102.68, 101.91, 101.37, 103.1, 104.6, 104.9, 
106.19, 108.2, 108.47, 106.28, 106.26, 107.45, 106.89, 108.16, 
106.31, 104.98, 103.1, 102.96, 103.38, 102.02, 100.01, 99.35, 
99.79, 99.37, 100.84, 102.21, 101.78, 102.64, 104.27, 103.5, 
104.7, 104.56, 106.38, 105.65, 107.12, 107.44, 106.11, 105.49, 
105.39, 106.64, 106.49, 106.11, 106.19, 105.87, 106.42, 108.08, 
106.08, 105.68, 105, 104.39, 104.39, 103.11, 103.58, 103.13, 
102.49, 101.33, 100.96, 100.28, 100.95, 101.47, 100.56, 100.91, 
100.28, 99.96, 98.88, 100.57, 100.99, 100.88, 101.42, 101.33, 
104, 105.04, 103.75, 105.34, 105.04, 106.46, 105.72, 104.54, 
104.44, 104.82, 106.26, 104.32, 104.02, 103.6, 104.27, 105.07, 
105.7, 105.75, 107.41, 108.39, 106.56, 106.51, 105.88, 105.88, 
104.77, 103.71, 103.8, 103.47, 104.75, 104.27, 104.84, 106.28, 
105.67, 104.61, 105.86, 102.84, 103.89, 105.27, 105.15, 104.1, 
103.32, 103, 103.64, 104.72, 104.25, 103.12, 102.11, 102.34, 
101.75, 100.93, 100.67, 100.27, 100.53, 100.03, 100, 101.25, 
101.87, 102.63, 101.94, 101.21, 101.39, 100.78, 100.8, 101.34, 
101.12, 100.96, 101.46, 102.6, 102.03, 144.3, 143.7, 142.92, 
143.29, 142.26, 142.61, 143.56, 143.01, 144.28, 142.1, 140.1, 
141.07, 141.94, 143.63, 142.45, 142.22, 140.5, 143.08, 143.13, 
142.39, 141.72, 142.2, 141.09, 139.01, 139.1, 138.17, 138.43, 
136.23, 138.25, 136.53, 134.98, 135.78, 136.79, 138.23, 138.59, 
136.68, 136.77, 136.87, 139.87, 141.3, 136.47, 137.86, 140.24, 
139.17, 137.77, 136.42, 134.46, 134.91, 131.76, 131.53, 132.82, 
131.83, 133.44, 130.35, 128.68, 131.5, 130.47, 128.66, 129.76, 
130.74, 131.76, 132.08, 132.38, 131.46, 128.49, 127.45, 126.49, 
125.12, 123.14, 122.97, 123.22, 120.75, 120.15, 119.25, 121.65, 
120.75, 119.99, 121.19, 122.41, 120.84, 121.02, 119.45, 117.18, 
117.97, 118.62, 115.37, 113.67, 114.67, 115.71, 117.94, 117.72, 
115.98, 115.86, 114.46, 116.9, 117.18, 118.51, 116.56, 118.86, 
119.33, 118.79, 119.08, 116.56, 116.22, 116.28, 116.03, 113.89, 
115.12, 114.1, 114.48, 113.77, 110.2, 108.81, 110.84, 111.72, 
112.08, 110.18, 113.21, 114.14, 112.56, 111.6, 112.12, 113.87, 
114.87, 115.82, 116.73, 114.77, 113.76, 112.5, 114.44, 115.12, 
116.44, 115.49, 116.14, 114.93, 114.31, 114.61, 116.59, 118.55, 
117.39, 117.48, 116.82, 117.75, 117.43, 120.04, 117.5, 119.35, 
119.31, 116.47, 118.35, 117.89, 117.12, 114.39, 115.55, 115.45, 
115.93, 114.72, 114.89, 113.04, 112.66, 112.04, 111.22, 112.77, 
113.62, 112.68, 112.96, 111.88, 109.6, 110.22, 107.49, 109.19, 
108.07, 108.31, 107.42, 107.67, 108.04, 107.97, 107.07, 107.32, 
106.25, 108.03, 105.27, 105.7, 103.68, 100.95, 104.33, 103.79, 
102.45, 103.83, 103.54, 102.55, 101.98, 100.99, 99.42, 99.86, 
100.35, 99.39, 98.57, 98.09, 98.32, 96.53, 94.88, 93.17, 92.4, 
95.09, 96.46, 97.47, 96.8, 96.11, 96.77, 96.35, 95.83, 94.22, 
93.28, 94.3, 94.61, 98.25, 97.89, 97.65, 99.23, 101.4, 100.33, 
99.64, 100.06, 101.48, 103.25, 102.9, 103.19, 101.98, 102.73, 
105.76, 107.09, 105.59, 106.95, 109.47, 109.17, 106.33, 106.67, 
107.7, 108.16, 108.32, 107.11, 105.31, 103.44, 103.7, 103.68, 
102.16, 100.14, 99.55, 100.68, 99.56, 101.46, 102.85, 103.28, 
105.18, 105.74, 103.99, 105.68, 107.76, 106.75, 106.06, 108.63, 
107.95, 106.2, 105.7, 107.22, 107.48, 106.98, 106.71, 106.32, 
107.17, 108.18, 108.32, 106.7, 105.91, 105.06, 105.56, 104.56, 
103.39, 104.26, 104.02, 103.05, 101.59, 102.38, 100.39, 101.31, 
102.3, 100.97, 101.25, 101.71, 100.37, 99.48, 101.89, 101.1, 
101.72, 101.82, 102.07, 105.59, 105.83, 104.09, 106.1, 105.27, 
106.85, 105.91, 105, 106.4, 106.4, 106.85, 104.34, 104.53, 104.32, 
104.8, 106.37, 106.6, 107.13, 108.25, 108.53, 106.78, 107, 106.89, 
106.05, 105.08, 103.83, 104.2, 104.1, 104.99, 104.4, 105.41, 
106.96, 105.82, 104.77, 106.46, 103.13, 105.7, 106.29, 105.24, 
104.89, 103.42, 103.05, 104.05, 105.15, 104.27, 103.19, 102.88, 
102.58, 101.82, 101.1, 100.74, 101.03, 100.53, 100.28, 101.09, 
102.1, 102.99, 103.12, 101.99, 101.8, 102.4, 100.88, 102.06, 
101.92, 102.22, 101.21, 102.02, 103, 102.6, 264907, 181524, 158021, 
156472, 148782, 116002, 157775, 123150, 122122, 153023, 155011, 
137588, 111926, 231710, 214209, 120633, 193896, 166147, 97075, 
104659, 159531, 153498, 130218, 180906, 103744, 139887, 120430, 
266719, 336580, 179902, 132917, 134601, 200799, 128252, 153603, 
130495, 222349, 119346, 193654, 304938, 228709, 150792, 209704, 
198956, 245816, 150546, 223650, 113040, 192219, 151020, 216364, 
157990, 348869, 207439, 177961, 154168, 131167, 187832, 144123, 
157318, 127214, 166782, 183352, 107455, 198180, 186263, 193479, 
211029, 220371, 184452, 141340, 263056, 152568, 214160, 252912, 
196477, 154411, 200698, 212299, 111813, 136024, 195180, 218625, 
154317, 218642, 386968, 251939, 223789, 285366, 271026, 224487, 
157493, 88949, 108094, 152604, 172372, 173089, 120480, 176159, 
111369, 117180, 94410, 214504, 117743, 80787, 114692, 159038, 
102300, 90694, 114591, 175295, 273106, 166334, 198349, 210769, 
168683, 120191, 155984, 169173, 144730, 86144, 115206, 89300, 
164764, 175206, 128895, 157766, 102323, 110939, 92882, 157034, 
151657, 121189, 104074, 120207, 71618, 69995, 106852, 138181, 
135332, 110882, 131150, 128856, 128361, 130054, 128637, 133136, 
89348, 136049, 127811, 61249, 117882, 147290, 87152, 59970, 88267, 
106138, 88903, 119465, 78274, 149826, 97575, 174577, 145673, 
109359, 114969, 105681, 164301, 94056, 177961, 125112, 114133, 
98800, 109987, 189028, 105675, 85228, 149171, 81864, 156813, 
182139, 219641, 186051, 183523, 272525, 284407, 151200, 210322, 
198712, 115838, 151126, 117095, 112434, 118249, 269890, 143414, 
248123, 136578, 153194, 137876, 177249, 204416, 351477, 243291, 
212182, 180463, 157366, 171678, 192957, 143684, 186724, 215348, 
178033, 111168, 160763, 133261, 326239, 114271, 108893, 208134, 
205780, 192292, 116300, 120259, 145580, 176567, 74634, 120283, 
128257, 174573, 361212, 193166, 159156, 169288, 235869, 110762, 
198551, 139224, 179036, 168863, 170507, 167068, 209428, 224791, 
137600, 94437, 122787, 166635, 139038, 130498, 173269, 202612, 
170440, 111057, 196028, 153819, 161347, 149160, 228776, 143152, 
109036, 287547, 117245, 135649, 102650, 117767, 91220, 116676, 
92403, 81988, 111697, 195935, 141713, 153559, 104531, 155391, 
111603, 143561, 134144, 111127, 121932, 119340, 136534, 145861, 
176203, 121991, 129529, 161432, 108378, 122268, 158744, 168612, 
220032, 137371, 189038, 173968, 179334, 348940, 428442, 322316, 
370209, 367544, 305517, 200126, 155634, 210179, 205960, 193401, 
167454, 139807, 136500, 111893, 165170, 144873, 146557, 177695, 
121582, 125835, 121101, 163632, 164461, 150864, 159463, 127293, 
117519, 119192, 115810, 122950, 179198, 160223, 135883, 180217, 
231170, 221326, 202181, 188773, 113898, 160268, 140702, 141496, 
133067, 108317, 150512, 203962, 137518, 179328, 195009, 156601, 
170262, 142783, 169191, 124908, 172794, 183648, 163245, 154917, 
116906, 106033, 199615, 194674, 179556, 162281, 204432, 209372, 
184100, 138847, 145.537, 143.386, 143.306, 142.845, 142.3165, 
142.085, 143.182, 143.4285, 143.7465, 142.5605, 140.539, 141.167, 
141.492, 143.438, 143.595, 142.581, 141.568, 142.9705, 142.96, 
142.446, 141.6615, 142.726, 140.889, 139.204, 138.9995, 138.239, 
138.7525, 136.836, 137.0375, 136.786, 135.3565, 135.307, 136.7935, 
137.85, 138.0885, 137.196, 137.789, 136.7305, 139.2625, 141.337, 
138.4795, 137.888, 140.245, 139.662, 137.766, 136.7355, 134.3375, 
134.738, 132.263, 131.963, 131.923, 132.093, 133.2055, 130.765, 
128.902, 130.1305, 130.5175, 128.688, 129.6505, 130.6365, 131.4075, 
132.3005, 131.8135, 131.4025, 128.948, 127.2585, 126.055, 124.9725, 
123.21, 123.411, 123.582, 121.1435, 120.192, 119.144, 120.9725, 
120.047, 120.307, 121.6665, 122.321, 121.271, 120.4945, 119.922, 
117.414, 118.614, 118.066, 115.0385, 113.978, 113.611, 116.1715, 
116.706, 118.2525, 116.386, 116.384, 114.8035, 116.2405, 117.607, 
118.4095, 117.147, 118.9845, 119.177, 118.476, 119.209, 116.7065, 
116.2555, 116.113, 115.439, 114.231, 115.2075, 114.4455, 114.26, 
113.7275, 110.472, 109.364, 109.738, 110.029, 112.073, 110.274, 
112.952, 114.371, 112.9545, 111.721, 111.755, 113.428, 115.325, 
116.1665, 117.076, 115.592, 114.163, 112.5225, 114.284, 115.664, 
115.546, 115.563, 116.243, 114.919, 114.4655, 114.584, 116.1025, 
118.469, 117.1855, 118.202, 117.799, 118.148, 118.305, 119.386, 
118.748, 118.1505, 119.012, 116.505, 118.091, 117.903, 117.485, 
115.117, 115.181, 116.1115, 115.226, 114.694, 115.11, 113.0305, 
112.593, 112.688, 111.301, 112.333, 113.1865, 112.469, 112.944, 
112.423, 109.6375, 110.044, 107.874, 108.8745, 108.708, 108.364, 
107.8725, 106.909, 107.765, 107.8615, 107.2055, 107.4455, 106.1255, 
107.2055, 105.3325, 105.554, 104.1995, 101.793, 103.782, 103.573, 
103.252, 103.7595, 103.8995, 102.306, 102.115, 101.028, 99.391, 
100.333, 100.1205, 99.442, 98.7555, 98.78, 97.914, 96.729, 95.441, 
93.3105, 92.7405, 94.84, 96.2215, 97.2825, 96.829, 95.8395, 96.6815, 
96.7765, 95.871, 94.7775, 93.817, 94.476, 94.298, 97.628, 97.9675, 
97.4475, 98.8485, 100.629, 100.243, 99.927, 100.208, 101.264, 
102.674, 102.626, 103.2325, 102.364, 102.1155, 104.684, 105.883, 
105.529, 106.82, 108.988, 108.975, 107.164, 106.881, 108.284, 
107.638, 108.607, 107.175, 105.39, 103.415, 103.647, 103.7805, 
102.372, 100.404, 99.831, 100.502, 99.841, 101.588, 102.783, 
102.771, 104.57, 105.2695, 104.0495, 105.152, 106.535, 106.9835, 
106.1915, 108.169, 107.9485, 106.515, 105.764, 106.805, 107.255, 
107.056, 106.66, 106.5115, 106.5215, 107.4785, 108.594, 106.671, 
105.932, 105.494, 105, 105.477, 103.5545, 104.0375, 103.913, 
103.0225, 101.8285, 101.903, 100.69, 101.3425, 102.0625, 101.079, 
101.3, 101.044, 100.406, 99.269, 101.29, 101.591, 101.742, 102.254, 
101.9195, 105.0865, 106.9935, 104.597, 106.74, 106.3425, 107.183, 
106.297, 104.907, 105.6185, 105.522, 106.798, 104.933, 104.366, 
104.151, 104.6915, 105.8815, 106.544, 107.247, 108.087, 108.618, 
106.952, 106.8115, 106.568, 106.474, 105.112, 103.951, 104.1915, 
103.856, 104.9465, 104.576, 105.265, 106.836, 106.274, 104.851, 
106.3515, 103.946, 105.1215, 106.058, 105.721, 104.606, 103.5795, 
103.295, 103.942, 105.063, 104.605, 103.2595, 102.489, 102.679, 
102.0055, 101.3165, 100.992, 100.741, 100.884, 100.386, 100.574, 
101.796, 102.565, 103.064, 102.2945, 101.905, 102.05, 101.24, 
101.7385, 101.8665, 101.7145, 101.384, 101.9365, 103.0215, 102.5165, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 87128, 83025, 71370, 66800, 63180, 
53709, 63831, 56450, 57580, 67809, 75131, 68763, 52611, 99037, 
96257, 68080, 102275, 79262, 49057, 54892, 67781, 67646, 64925, 
79020, 50020, 59748, 57051, 132469, 167653, 94922, 67696, 69346, 
94254, 64239, 75960, 66708, 131199, 66224, 104842, 159550, 128274, 
81398, 103154, 125086, 124362, 88850, 115527, 64734, 94998, 82646, 
126878, 86730, 195925, 106761, 93431, 82660, 66866, 108651, 74096, 
84234, 72783, 89679, 96548, 60650, 108123, 111928, 108583, 107834, 
111782, 99558, 74504, 144936, 73498, 120431, 119767, 112801, 
89596, 107763, 125136, 70576, 80106, 115422, 107201, 86164, 126337, 
191505, 125640, 108330, 138789, 132828, 111715, 71540, 42768, 
49205, 69044, 80932, 74009, 52740, 79124, 52173, 55571, 40965, 
91781, 59848, 41787, 52293, 67686, 49468, 38111, 48677, 84133, 
128182, 83307, 115009, 95621, 83950, 55992, 76338, 85223, 63388, 
45002, 53196, 45669, 76859, 87562, 61112, 75202, 47388, 49222, 
41812, 47912, 76397, 61070, 51510, 50126, 36750, 35418, 52811, 
69231, 64648, 56870, 71962, 64629, 64683, 63731, 65563, 61285, 
43095, 65781, 54728, 33692, 58168, 76581, 47217, 32535, 43173, 
47405, 44295, 51750, 37310, 61341, 43367, 57544, 62864, 44470, 
49262, 52327, 76789, 48689, 80981, 55617, 48336, 43691, 50776, 
77308, 48469, 39410, 55794, 39903, 58663, 88435, 108972, 90462, 
84994, 135362, 143205, 74608, 106289, 91554, 56702, 74494, 52881, 
52357, 55584, 83703, 75937, 106116, 71380, 72583, 67749, 79769, 
97960, 177957, 118893, 97779, 92745, 82954, 85473, 86505, 69751, 
100443, 94404, 85458, 50292, 61856, 66470, 146169, 50370, 52605, 
97626, 98874, 91861, 60353, 59826, 68320, 85196, 34029, 52246, 
62267, 81201, 146515, 89768, 66700, 75819, 99871, 52438, 78986, 
57361, 82702, 84934, 71629, 71025, 79399, 90792, 54552, 42079, 
47637, 60735, 53166, 51322, 74548, 81794, 82556, 53636, 89216, 
66955, 75873, 65508, 111210, 70645, 52020, 116226, 56272, 68489, 
53070, 59622, 47809, 51991, 40618, 38418, 56083, 92334, 72220, 
70429, 46552, 65421, 55246, 71449, 65706, 52461, 62908, 62737, 
66927, 68181, 79280, 56136, 67787, 70700, 53682, 65618, 78957, 
81937, 95354, 67438, 93078, 80989, 85664, 182388, 232848, 190021, 
217963, 225282, 165791, 121024, 87929, 109956, 113423, 108769, 
82210, 69674, 71590, 56898, 84999, 71827, 70761, 92091, 61148, 
58973, 59886, 83802, 78083, 70235, 66859, 55006, 56000, 55055, 
57424, 53648, 89294, 75554, 65229, 84780, 97266, 110277, 97189, 
106115, 54251, 72610, 65440, 67455, 72810, 58519, 74706, 85607, 
71704, 80982, 85008, 71274, 77452, 68270, 83407, 62892, 78075, 
87120, 69637, 65342, 57496, 47581, 90824, 71466, 63885, 72290, 
88520, 91585, 80649, 65101), class = c("xts", "zoo"), .indexCLASS = "Date", .indexTZ = "", tclass = c("POSIXct", 
"POSIXt"), tzone = "", from = "20160516  23:59:59", to = "20170516  23:59:59", src = "IB", updated = structure(1495037082.30338, class = c("POSIXct", 
"POSIXt")), index = structure(c(1641186000, 1641272400, 1641358800, 
1641445200, 1641531600, 1641790800, 1641877200, 1641963600, 1642050000, 
1642136400, 1642482000, 1642568400, 1642654800, 1642741200, 1643000400, 
1643086800, 1643173200, 1643259600, 1643346000, 1643605200, 1643691600, 
1643778000, 1643864400, 1643950800, 1644210000, 1644296400, 1644382800, 
1644469200, 1644555600, 1644814800, 1644901200, 1644987600, 1645074000, 
1645160400, 1645506000, 1645592400, 1645678800, 1645765200, 1646024400, 
1646110800, 1646197200, 1646283600, 1646370000, 1646629200, 1646715600, 
1646802000, 1646888400, 1646974800, 1647230400, 1647316800, 1647403200, 
1647489600, 1647576000, 1647835200, 1647921600, 1648008000, 1648094400, 
1648180800, 1648440000, 1648526400, 1648612800, 1648699200, 1648785600, 
1649044800, 1649131200, 1649217600, 1649304000, 1649390400, 1649649600, 
1649736000, 1649822400, 1649908800, 1650254400, 1650340800, 1650427200, 
1650513600, 1650600000, 1650859200, 1650945600, 1651032000, 1651118400, 
1651204800, 1651464000, 1651550400, 1651636800, 1651723200, 1651809600, 
1652068800, 1652155200, 1652241600, 1652328000, 1652414400, 1652673600, 
1652760000, 1652846400, 1652932800, 1653019200, 1653278400, 1653364800, 
1653451200, 1653537600, 1653624000, 1653969600, 1654056000, 1654142400, 
1654228800, 1654488000, 1654574400, 1654660800, 1654747200, 1654833600, 
1655092800, 1655179200, 1655265600, 1655352000, 1655438400, 1655784000, 
1655870400, 1655956800, 1656043200, 1656302400, 1656388800, 1656475200, 
1656561600, 1656648000, 1656993600, 1657080000, 1657166400, 1657252800, 
1657512000, 1657598400, 1657684800, 1657771200, 1657857600, 1658116800, 
1658203200, 1658289600, 1658376000, 1658462400, 1658721600, 1658808000, 
1658894400, 1658980800, 1659067200, 1659326400, 1659412800, 1659499200, 
1659585600, 1659672000, 1659931200, 1660017600, 1660104000, 1660190400, 
1660276800, 1660536000, 1660622400, 1660708800, 1660795200, 1660881600, 
1661140800, 1661227200, 1661313600, 1661400000, 1661486400, 1661745600, 
1661832000, 1661918400, 1662004800, 1662091200, 1662436800, 1662523200, 
1662609600, 1662696000, 1662955200, 1663041600, 1663128000, 1663214400, 
1663300800, 1663560000, 1663646400, 1663732800, 1663819200, 1663905600, 
1664164800, 1664251200, 1664337600, 1664424000, 1664510400, 1664769600, 
1664856000, 1664942400, 1665028800, 1665115200, 1665374400, 1665460800, 
1665547200, 1665633600, 1665720000, 1665979200, 1666065600, 1666152000, 
1666238400, 1666324800, 1666584000, 1666670400, 1666756800, 1666843200, 
1666929600, 1667188800, 1667275200, 1667361600, 1667448000, 1667534400, 
1667797200, 1667883600, 1667970000, 1668056400, 1668142800, 1668402000, 
1668488400, 1668574800, 1668661200, 1668747600, 1669006800, 1669093200, 
1669179600, 1669352400, 1669611600, 1669698000, 1669784400, 1669870800, 
1669957200, 1670216400, 1670302800, 1670389200, 1670475600, 1670562000, 
1670821200, 1670907600, 1670994000, 1671080400, 1671166800, 1671426000, 
1671512400, 1671598800, 1671685200, 1671771600, 1672117200, 1672203600, 
1672290000, 1672376400, 1672722000, 1672808400, 1672894800, 1672981200, 
1673240400, 1673326800, 1673413200, 1673499600, 1673586000, 1673931600, 
1674018000, 1674104400, 1674190800, 1674450000, 1674536400, 1674622800, 
1674709200, 1674795600, 1675054800, 1675141200, 1675227600, 1675314000, 
1675400400, 1675659600, 1675746000, 1675832400, 1675918800, 1676005200, 
1676264400, 1676350800, 1676437200, 1676523600, 1676610000, 1676955600, 
1677042000, 1677128400, 1677214800, 1677474000, 1677560400, 1677646800, 
1677733200, 1677819600, 1678078800, 1678165200, 1678251600, 1678338000, 
1678424400, 1678680000, 1678766400, 1678852800, 1678939200, 1679025600, 
1679284800, 1679371200, 1679457600, 1679544000, 1679630400, 1679889600, 
1679976000, 1680062400, 1680148800, 1680235200, 1680494400, 1680580800, 
1680667200, 1680753600, 1681099200, 1681185600, 1681272000, 1681358400, 
1681444800, 1681704000, 1681790400, 1681876800, 1681963200, 1682049600, 
1682308800, 1682395200, 1682481600, 1682568000, 1682654400, 1682913600, 
1.683e+09, 1683086400, 1683172800, 1683259200, 1683518400, 1683604800, 
1683691200, 1683777600, 1683864000, 1684123200, 1684209600, 1684296000, 
1684382400, 1684468800, 1684728000, 1684814400, 1684900800, 1684987200, 
1685073600, 1685419200, 1685505600, 1685592000, 1685678400, 1685937600, 
1686024000, 1686110400, 1686196800, 1686283200, 1686542400, 1686628800, 
1686715200, 1686801600, 1686888000), tzone = "UTC", tclass = "Date"), dim = c(366L, 
8L), dimnames = list(NULL, c("TLT.Open", "TLT.High", "TLT.Low", 
"TLT.Close", "TLT.Volume", "TLT.WAP", "TLT.hasGaps", "TLT.Count"
)))

> dput(lastNthTradingDaysOfMonth)
structure(c(19023, 19051, 19082, 19111, 19143, 19173, 19202, 
19235, 19265, 19296, 19326, 19356, 19388, 19416, 19447, 19475, 
19508), class = "Date")

> series$IsNthTradingDayOfMonth <- 0
> series$IsNthTradingDayOfMonth[lastNthTradingDaysOfMonth] <- 1
> unique(series$IsNthTradingDayOfMonth)
[1] 0
> series$IsNthTradingDayOfMonth[which(index(series) %in% lastNthTradingDaysOfMonth)] <- 1
> unique(series$IsNthTradingDayOfMonth)
[1] 0 1

Session Info

[Insert your sessionInfo() output]
joshuaulrich commented 1 year ago

The index values of series are not actually seconds at the end of the days in UTC. They're off by 5 hours.

y <- head(series)
x <- y
index(x) <- as.Date(index(x))
(.index(y) - .index(x)) / 3600  # difference of 5 hours
## [1] 5 5 5 5 5 5
## attr(,"tzone")
## [1] "UTC"
## attr(,"tclass")
## [1] "Date"

How did you create series? It's in an old format because xts indexes no longer have .indexClass and .indexTZ attributes. Also tclass and tclass attributes are only on the index now, not the xts object itself.

SamoPP commented 1 year ago

Thanks. I was assuming some tzone nightmare was happening....

Well, I created the xts object many years ago by downloading available history at that time from Interactive Brokers using IBrokers, saved it to disk and then (almost) every day append to it what is new (from IB) and saved the appended xts to disk. This was read from disk using getSymbols.FI.

joshuaulrich commented 1 year ago

In that case, I would add a index(x) <- as.Date(index(x)) to your code that downloads the data from IB and appends it to the old object.

It may make sense to check and warn if the underlying index are not UTC dates when the tclass = "Date" and tzone = "UTC". Can you share some of the code you're using to create the data you're appending? That would help me determine where it might make sense to add a check.

Also, you can use x <- xts:::.update_index_attributes(x) to update the tclass and tzone attributes to use the current structure.

SamoPP commented 1 year ago

Thanks. Will do as suggested.

Below is the code (well, part of it - it's a big mess) that downloads and updates data from IB and updates xts object on disk:


instr <- getInstrument(symbol)

contract <- instr$IB
symbol <- instr$primary_id

startTimeForOneSymbol <- Sys.time()
i <- i + 1

if (averageTimeTakenForOneSymbol != 0) {
  print(paste0("[INFO] START - Download from IB for symbol=", symbol, " i=", i, " of ", nSymbols, " . Estimated time left: ", averageTimeTakenForOneSymbol*(nSymbols - i + 1)))
} else {
  print(paste0("[INFO] START - Download from IB for symbol=", symbol, " i=", i, " of ", nSymbols))
}

firstDateOnDisk <- NULL
dateOfFirstTrade <- NULL
bizDaysFromHeadTimeStampToFirstDayOnDisk <- NULL
existingSymbolDataOnDisk <- NULL
print(paste0("[INFO] Retreiving existing data for symbol=", symbol, " from disk folder=", file.path(pathToIBData, "TRADES")))
try(existingSymbolDataOnDisk <- getSymbols(symbol, src="FI", env=env, dir=file.path(pathToIBData, "TRADES"), auto.assign=FALSE, extension="rda", split_method="common"))

if (!is.null(existingSymbolDataOnDisk) & !inherits(existingSymbolDataOnDisk, "try-error")) {
  if (tclass(existingSymbolDataOnDisk) != "Date") {
    tclass(existingSymbolDataOnDisk) <- "Date"
  }

  lastDateOnDisk <- last(index(existingSymbolDataOnDisk))
  nDaysToDownload <- businessDaysBetween("UnitedStates/NYSE", from=lastDateOnDisk, to=lastNYSEOpenDate, includeFirst=FALSE, includeLast=TRUE)
  print(paste0("[INFO] We have existing data for symbol=", symbol, " in folder=", file.path(pathToIBData, "TRADES"), " with lastDateOnDisk=", lastDateOnDisk, " therefore using nDaysToDownload=", nDaysToDownload))
} else {
  if (isTRUE(usereqHeadTimestamp)) {
    # Do the reqHeadTimestamp thing to find nDaysToDownload
    if (is.null(contract)) {
      contract <- Contr_From_Instr(symbol)
    }

    #dateOfFirstTrade <- try(reqHeadTimeStamp(contract, insync=insync, ib=ib, maxAttempts=5))
    dateOfFirstTrade <- try(getAndManageReqHeadTimeStamp(symbol=symbol, contract=contract, insync=insync, ib=ib, maxAttempts=5, tryLoadingFromDisk=TRUE, folder="/data_warehouse/data/sources/ib/HeadTimeStamp/"))

    if (!is.null(dateOfFirstTrade) & !inherits(dateOfFirstTrade, "try-error") & (length(dateOfFirstTrade) > 0)) {
      if (dateOfFirstTrade > 0) {
        dateOfFirstTrade <- as.Date(dateOfFirstTrade, origin="1970-01-01")

        print(paste0("[INFO] First trade for symbol=", symbol, " appeared on ", dateOfFirstTrade))
      } else {
        print(paste0("[INFO] First trade for symbol=", symbol, " is a negative number (dateOfFirstTrade=", dateOfFirstTrade, ") and therefore probably data does not exist. Setting dateOfFirstTrade <- NULL ."))

        dateOfFirstTrade <- NULL
      }
    } else {
      print(paste0("[INFO] Could not get date of first trade for symbol=", symbol, " . There was an eror calling reqHeadTimeStamp()."))

      dateOfFirstTrade <- NULL
    }

    if (!is.null(dateOfFirstTrade)) {
      nDaysToDownload <- max(maxDays, businessDaysBetween("UnitedStates/NYSE", from=dateOfFirstTrade, to=lastNYSEOpenDate, includeFirst=FALSE, includeLast=TRUE))
    } else {
      nDaysToDownload <- maxDays
      print(paste0("[INFO] For symbol=", symbol, " dateOfFirstTrade is NULL. There was an error or something instead. Maybe data simply just doesn't exist yet. Setting nDaysToDownload <- ", maxDays, " (maxDays) ."))
    }

    print(paste0("[INFO] We do not have existing data for symbol=", symbol, " in folder=", file.path(pathToIBData, "TRADES"), " therefore using nDaysToDownload=", nDaysToDownload, " (based on reqHeadTimestamp)."))
  } else {
    nDaysToDownload <- maxDays
    print(paste0("[INFO] We do not have existing data for symbol=", symbol, " in folder=", file.path(pathToIBData, "TRADES"), " therefore using nDaysToDownload=", nDaysToDownload, " (maxDays)."))
  }
}

if (nDaysToDownload > 0) {
  newSymbolData <- NULL
  endDateTime <- paste(format(strptime(paste(to, "23:59:59"),"%Y-%m-%d %H:%M:%S"),"%Y%m%d %H:%M:%S"))

  if (nDaysToDownload <= 365) {
    duration <- paste0(nDaysToDownload, " D")
  } else {
    duration <- paste0(ceiling(nDaysToDownload/365), " Y")
  }

  if (is.null(contract)) {
    contract <- Contr_From_Instr(symbol)
  }

  if (is.null(tws)) {
    tws <- twsConnect(clientId=5)
  }

  print(paste0("[INFO] Retreiving IB data for symbol=", symbol, " with duration=", duration, ", endDateTime=", endDateTime, " ."))

  newSymbolData <- try(reqHistoricalData(tws, contract, barSize="1 day", duration=duration, endDateTime=endDateTime, useRTH="1", whatToShow="TRADES"))

  if (!is.null(newSymbolData) & !inherits(newSymbolData, "try-error")) {
    print(paste0("[INFO] Received new data from IB for symbol=", symbol, " and we have NROW(newSymbolData)=", NROW(newSymbolData), " ."))
    tclass(newSymbolData) <- "Date"

    combinedSymbolData <- NULL
    if (!is.null(existingSymbolDataOnDisk)) {
      combinedSymbolData <- rbind(existingSymbolDataOnDisk, newSymbolData)
    } else {
      combinedSymbolData <- newSymbolData
    }

    # Remove rows with a duplicated timestamp, but keep the latest one
    combinedSymbolData <- combinedSymbolData[!duplicated(index(combinedSymbolData), fromLast=TRUE ), ]

    assign(symbol, combinedSymbolData, envir=env)

    saveSymbols.common(symbol, base_dir=file.path(pathToIBData, "TRADES"), env=env)
    print(paste0("[INFO] Saved symbol=", symbol, " to folder=", file.path(pathToIBData, "TRADES")))
  } else {
    print(paste0("[INFO] Not able to receive new data from IB for symbol=", symbol))
  }
} else {
  print(paste0("[INFO] Nothing to download from IB for symbol=", symbol, " since nDaysToDownload=", nDaysToDownload, " (shoul be nDaysToDownload > 0)."))
}
joshuaulrich commented 1 year ago

The problem is this block of code. It changes the tclass attribute to "Date" but it doesn't update the index to seconds since the epoch in UTC.

if (tclass(existingSymbolDataOnDisk) != "Date") {
    tclass(existingSymbolDataOnDisk) <- "Date"
}

This is similar enough to #311 that I'm going to close it as a duplicate. Please continue any discussions in that issue.