Skip to content

Commit 0da2ba9

Browse files
sethmlarsonsethmlarson
authored
Set json/x-ndjson serializer to compatibility mode mimetype too
1 parent dc1d7cd commit 0da2ba9

File tree

11 files changed

+267
-48
lines changed

11 files changed

+267
-48
lines changed

‎elasticsearch/_async/client/__init__.py

+14-1
Original file line numberDiff line numberDiff line change
@@ -352,9 +352,22 @@ def __init__(
352352
if meta_header is not DEFAULT:
353353
transport_kwargs["meta_header"] = meta_header
354354

355-
transport_serializers = DEFAULT_SERIALIZERS
355+
transport_serializers = DEFAULT_SERIALIZERS.copy()
356356
if serializers is not DEFAULT:
357357
transport_serializers.update(serializers)
358+
359+
# Override compatibility serializers from their non-compat mimetypes too.
360+
# So we use the same serializer for requests and responses.
361+
for mime_subtype in ("json", "x-ndjson"):
362+
if f"application/{mime_subtype}" in serializers:
363+
compat_mimetype = (
364+
f"application/vnd.elasticsearch+{mime_subtype}"
365+
)
366+
if compat_mimetype not in serializers:
367+
transport_serializers[compat_mimetype] = serializers[
368+
f"application/{mime_subtype}"
369+
]
370+
358371
transport_kwargs["serializers"] = transport_serializers
359372

360373
transport_kwargs["default_mimetype"] = default_mimetype

‎elasticsearch/_async/client/_base.py

+22-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
)
4545
from elastic_transport.client_utils import DEFAULT, DefaultType
4646

47+
from ..._version import __versionstr__
4748
from ...compat import warn_stacklevel
4849
from ...exceptions import (
4950
HTTP_EXCEPTIONS,
@@ -56,6 +57,11 @@
5657
from .utils import _TYPE_ASYNC_SNIFF_CALLBACK, _base64_auth_header, _quote_query
5758

5859
_WARNING_RE = re.compile(r"\"([^\"]*)\"")
60+
_COMPAT_MIMETYPE_TEMPLATE = "application/vnd.elasticsearch+%s; compatible-with=" + str(
61+
__versionstr__.partition(".")[0]
62+
)
63+
_COMPAT_MIMETYPE_RE = re.compile(r"application/(json|x-ndjson|vnd\.mapbox-vector-tile)")
64+
_COMPAT_MIMETYPE_SUB = _COMPAT_MIMETYPE_TEMPLATE % (r"\g<1>",)
5965

6066

6167
def resolve_auth_headers(
@@ -166,7 +172,9 @@ async def sniff_callback(
166172
meta, node_infos = await transport.perform_request(
167173
"GET",
168174
"/_nodes/_all/http",
169-
headers={"accept": "application/json"},
175+
headers={
176+
"accept": "application/vnd.elasticsearch+json; compatible-with=8"
177+
},
170178
request_timeout=(
171179
sniff_options.sniff_timeout
172180
if not sniff_options.is_initial_sniff
@@ -257,6 +265,19 @@ async def perform_request(
257265
else:
258266
request_headers = self._headers
259267

268+
def mimetype_header_to_compat(header: str) -> None:
269+
# Converts all parts of a Accept/Content-Type headers
270+
# from application/X -> application/vnd.elasticsearch+X
271+
nonlocal request_headers
272+
mimetype = request_headers.get(header, None)
273+
if mimetype:
274+
request_headers[header] = _COMPAT_MIMETYPE_RE.sub(
275+
_COMPAT_MIMETYPE_SUB, mimetype
276+
)
277+
278+
mimetype_header_to_compat("Accept")
279+
mimetype_header_to_compat("Content-Type")
280+
260281
if params:
261282
target = f"{path}?{_quote_query(params)}"
262283
else:

‎elasticsearch/_sync/client/__init__.py

+14-1
Original file line numberDiff line numberDiff line change
@@ -352,9 +352,22 @@ def __init__(
352352
if meta_header is not DEFAULT:
353353
transport_kwargs["meta_header"] = meta_header
354354

355-
transport_serializers = DEFAULT_SERIALIZERS
355+
transport_serializers = DEFAULT_SERIALIZERS.copy()
356356
if serializers is not DEFAULT:
357357
transport_serializers.update(serializers)
358+
359+
# Override compatibility serializers from their non-compat mimetypes too.
360+
# So we use the same serializer for requests and responses.
361+
for mime_subtype in ("json", "x-ndjson"):
362+
if f"application/{mime_subtype}" in serializers:
363+
compat_mimetype = (
364+
f"application/vnd.elasticsearch+{mime_subtype}"
365+
)
366+
if compat_mimetype not in serializers:
367+
transport_serializers[compat_mimetype] = serializers[
368+
f"application/{mime_subtype}"
369+
]
370+
358371
transport_kwargs["serializers"] = transport_serializers
359372

360373
transport_kwargs["default_mimetype"] = default_mimetype

‎elasticsearch/_sync/client/_base.py

+22-1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
)
4545
from elastic_transport.client_utils import DEFAULT, DefaultType
4646

47+
from ..._version import __versionstr__
4748
from ...compat import warn_stacklevel
4849
from ...exceptions import (
4950
HTTP_EXCEPTIONS,
@@ -56,6 +57,11 @@
5657
from .utils import _TYPE_SYNC_SNIFF_CALLBACK, _base64_auth_header, _quote_query
5758

5859
_WARNING_RE = re.compile(r"\"([^\"]*)\"")
60+
_COMPAT_MIMETYPE_TEMPLATE = "application/vnd.elasticsearch+%s; compatible-with=" + str(
61+
__versionstr__.partition(".")[0]
62+
)
63+
_COMPAT_MIMETYPE_RE = re.compile(r"application/(json|x-ndjson|vnd\.mapbox-vector-tile)")
64+
_COMPAT_MIMETYPE_SUB = _COMPAT_MIMETYPE_TEMPLATE % (r"\g<1>",)
5965

6066

6167
def resolve_auth_headers(
@@ -166,7 +172,9 @@ def sniff_callback(
166172
meta, node_infos = transport.perform_request(
167173
"GET",
168174
"/_nodes/_all/http",
169-
headers={"accept": "application/json"},
175+
headers={
176+
"accept": "application/vnd.elasticsearch+json; compatible-with=8"
177+
},
170178
request_timeout=(
171179
sniff_options.sniff_timeout
172180
if not sniff_options.is_initial_sniff
@@ -257,6 +265,19 @@ def perform_request(
257265
else:
258266
request_headers = self._headers
259267

268+
def mimetype_header_to_compat(header: str) -> None:
269+
# Converts all parts of a Accept/Content-Type headers
270+
# from application/X -> application/vnd.elasticsearch+X
271+
nonlocal request_headers
272+
mimetype = request_headers.get(header, None)
273+
if mimetype:
274+
request_headers[header] = _COMPAT_MIMETYPE_RE.sub(
275+
_COMPAT_MIMETYPE_SUB, mimetype
276+
)
277+
278+
mimetype_header_to_compat("Accept")
279+
mimetype_header_to_compat("Content-Type")
280+
260281
if params:
261282
target = f"{path}?{_quote_query(params)}"
262283
else:

‎elasticsearch/serializer.py

+7-21
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
from elastic_transport import Serializer as Serializer
2626
from elastic_transport import TextSerializer as TextSerializer
2727

28-
from .compat import to_bytes
2928
from .exceptions import SerializationError
3029

3130
INTEGER_TYPES = ()
@@ -37,7 +36,8 @@
3736
"JsonSerializer",
3837
"TextSerializer",
3938
"NdjsonSerializer",
40-
"CompatibilityModeSerializer",
39+
"CompatibilityModeJsonSerializer",
40+
"CompatibilityModeNdjsonSerializer",
4141
"MapboxVectorTileSerializer",
4242
]
4343

@@ -80,27 +80,12 @@ def default(self, data: Any) -> Any:
8080
return JsonSerializer.default(self, data)
8181

8282

83-
class CompatibilityModeSerializer(JsonSerializer):
83+
class CompatibilityModeJsonSerializer(JsonSerializer):
8484
mimetype: ClassVar[str] = "application/vnd.elasticsearch+json"
8585

86-
def dumps(self, data: Any) -> bytes:
87-
if isinstance(data, str):
88-
data = data.encode("utf-8", "surrogatepass")
89-
if isinstance(data, bytes):
90-
return data
91-
if isinstance(data, (tuple, list)):
92-
return NdjsonSerializer.dumps(self, data) # type: ignore
93-
return JsonSerializer.dumps(self, data)
9486

95-
def loads(self, data: bytes) -> Any:
96-
if isinstance(data, str):
97-
data = to_bytes(data, "utf-8")
98-
if isinstance(data, bytes) and data.endswith(b"\n"):
99-
return NdjsonSerializer.loads(self, data) # type: ignore
100-
try: # Try as JSON first but if that fails then try NDJSON.
101-
return JsonSerializer.loads(self, data)
102-
except SerializationError:
103-
return NdjsonSerializer.loads(self, data) # type: ignore
87+
class CompatibilityModeNdjsonSerializer(NdjsonSerializer):
88+
mimetype: ClassVar[str] = "application/vnd.elasticsearch+x-ndjson"
10489

10590

10691
class MapboxVectorTileSerializer(Serializer):
@@ -119,7 +104,8 @@ def dumps(self, data: bytes) -> bytes:
119104
JsonSerializer.mimetype: JsonSerializer(),
120105
MapboxVectorTileSerializer.mimetype: MapboxVectorTileSerializer(),
121106
NdjsonSerializer.mimetype: NdjsonSerializer(),
122-
CompatibilityModeSerializer.mimetype: CompatibilityModeSerializer(),
107+
CompatibilityModeJsonSerializer.mimetype: CompatibilityModeJsonSerializer(),
108+
CompatibilityModeNdjsonSerializer.mimetype: CompatibilityModeNdjsonSerializer(),
123109
}
124110

125111
# Alias for backwards compatibility

‎test_elasticsearch/test_async/test_transport.py

+9-5
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ async def test_client_meta_header_not_sent(self):
243243
calls = client.transport.node_pool.get().calls
244244
assert 1 == len(calls)
245245
assert calls[0][1]["headers"] == {
246-
"accept": "application/json",
246+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
247247
}
248248

249249
async def test_body_surrogates_replaced_encoded_into_bytes(self):
@@ -391,7 +391,9 @@ async def test_sniff_on_start_ignores_sniff_timeout(self):
391391
("GET", "/_nodes/_all/http"),
392392
{
393393
"body": None,
394-
"headers": {"accept": "application/json"},
394+
"headers": {
395+
"accept": "application/vnd.elasticsearch+json; compatible-with=8"
396+
},
395397
"request_timeout": None, # <-- Should be None instead of 12
396398
},
397399
)
@@ -418,7 +420,7 @@ async def test_sniff_uses_sniff_timeout(self):
418420
{
419421
"body": None,
420422
"headers": {
421-
"accept": "application/json",
423+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
422424
},
423425
"request_timeout": DEFAULT,
424426
},
@@ -427,7 +429,9 @@ async def test_sniff_uses_sniff_timeout(self):
427429
("GET", "/_nodes/_all/http"),
428430
{
429431
"body": None,
430-
"headers": {"accept": "application/json"},
432+
"headers": {
433+
"accept": "application/vnd.elasticsearch+json; compatible-with=8"
434+
},
431435
"request_timeout": 12,
432436
},
433437
)
@@ -681,7 +685,7 @@ async def test_unsupported_product_error(headers):
681685
{
682686
"body": None,
683687
"headers": {
684-
"accept": "application/json",
688+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
685689
},
686690
"request_timeout": DEFAULT,
687691
},

‎test_elasticsearch/test_client/test_deprecated_options.py

+2
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ class CustomSerializer(JsonSerializer):
135135
"application/json",
136136
"text/*",
137137
"application/vnd.elasticsearch+json",
138+
"application/vnd.elasticsearch+x-ndjson",
138139
}
139140

140141
client = Elasticsearch(
@@ -154,5 +155,6 @@ class CustomSerializer(JsonSerializer):
154155
"application/json",
155156
"text/*",
156157
"application/vnd.elasticsearch+json",
158+
"application/vnd.elasticsearch+x-ndjson",
157159
"application/cbor",
158160
}

‎test_elasticsearch/test_client/test_options.py

+13-13
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ def test_options_passed_to_perform_request(self):
139139
assert call.pop("client_meta") is DEFAULT
140140
assert call == {
141141
"headers": {
142-
"accept": "application/json",
142+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
143143
},
144144
"body": None,
145145
}
@@ -157,7 +157,7 @@ def test_options_passed_to_perform_request(self):
157157
assert call.pop("client_meta") is DEFAULT
158158
assert call == {
159159
"headers": {
160-
"accept": "application/json",
160+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
161161
},
162162
"body": None,
163163
"request_timeout": 1,
@@ -182,7 +182,7 @@ def test_options_passed_to_perform_request(self):
182182
assert call.pop("client_meta") is DEFAULT
183183
assert call == {
184184
"headers": {
185-
"accept": "application/json",
185+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
186186
},
187187
"body": None,
188188
"request_timeout": 1,
@@ -209,7 +209,7 @@ async def test_options_passed_to_async_perform_request(self):
209209
assert call.pop("client_meta") is DEFAULT
210210
assert call == {
211211
"headers": {
212-
"accept": "application/json",
212+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
213213
},
214214
"body": None,
215215
}
@@ -227,7 +227,7 @@ async def test_options_passed_to_async_perform_request(self):
227227
assert call.pop("client_meta") is DEFAULT
228228
assert call == {
229229
"headers": {
230-
"accept": "application/json",
230+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
231231
},
232232
"body": None,
233233
"request_timeout": 1,
@@ -252,7 +252,7 @@ async def test_options_passed_to_async_perform_request(self):
252252
assert call.pop("client_meta") is DEFAULT
253253
assert call == {
254254
"headers": {
255-
"accept": "application/json",
255+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
256256
},
257257
"body": None,
258258
"request_timeout": 1,
@@ -294,7 +294,7 @@ def test_http_headers_overrides(self):
294294

295295
assert call["headers"] == {
296296
"key": "val",
297-
"accept": "application/json",
297+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
298298
}
299299

300300
client.options(headers={"key1": "val"}).indices.get(index="2")
@@ -303,15 +303,15 @@ def test_http_headers_overrides(self):
303303
assert call["headers"] == {
304304
"key": "val",
305305
"key1": "val",
306-
"accept": "application/json",
306+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
307307
}
308308

309309
client.options(headers={"key": "val2"}).indices.get(index="3")
310310
call = calls[("GET", "/3")][0]
311311

312312
assert call["headers"] == {
313313
"key": "val2",
314-
"accept": "application/json",
314+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
315315
}
316316

317317
client = Elasticsearch(
@@ -338,14 +338,14 @@ def test_user_agent_override(self):
338338
call = calls[("GET", "/1")][0]
339339
assert call["headers"] == {
340340
"user-agent": "custom1",
341-
"accept": "application/json",
341+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
342342
}
343343

344344
client.indices.get(index="2", headers={"user-agent": "custom2"})
345345
call = calls[("GET", "/2")][0]
346346
assert call["headers"] == {
347347
"user-agent": "custom2",
348-
"accept": "application/json",
348+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
349349
}
350350

351351
client = Elasticsearch(
@@ -359,12 +359,12 @@ def test_user_agent_override(self):
359359
call = calls[("GET", "/1")][0]
360360
assert call["headers"] == {
361361
"user-agent": "custom3",
362-
"accept": "application/json",
362+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
363363
}
364364

365365
client.indices.get(index="2", headers={"user-agent": "custom4"})
366366
call = calls[("GET", "/2")][0]
367367
assert call["headers"] == {
368368
"user-agent": "custom4",
369-
"accept": "application/json",
369+
"accept": "application/vnd.elasticsearch+json; compatible-with=8",
370370
}

0 commit comments

Comments
 (0)