Skip to content

src: move all 1-byte encodings to native #61118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChALkeR wants to merge 1 commit into
base: main
Choose a base branch
from
+542 −160
Open

src: move all 1-byte encodings to native #61118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 47 additions & 4 deletions lib/internal/encoding.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@
// https://encoding.spec.whatwg.org

const {
ArrayPrototypeMap,
Boolean,
ObjectDefineProperties,
ObjectGetOwnPropertyDescriptors,
ObjectSetPrototypeOf,
ObjectValues,
SafeArrayIterator,
SafeMap,
StringPrototypeSlice,
Symbol,
Expand All @@ -32,8 +34,6 @@ const kFatal = Symbol('kFatal');
const kUTF8FastPath = Symbol('kUTF8FastPath');
const kIgnoreBOM = Symbol('kIgnoreBOM');

const { isSinglebyteEncoding, createSinglebyteDecoder } = require('internal/encoding/single-byte');

const {
getConstructorOf,
customInspectSymbol: inspect,
Expand All @@ -58,6 +58,7 @@ const {
encodeIntoResults,
encodeUtf8String,
decodeUTF8,
decodeSingleByte,
} = binding;

function validateDecoder(obj) {
Expand All @@ -71,6 +72,47 @@ const CONVERTER_FLAGS_IGNORE_BOM = 0x4;

const empty = new FastBuffer();

// Has to be synced with src/
const encodingsSinglebyte = new SafeMap(new SafeArrayIterator(ArrayPrototypeMap([
'ibm866',
'koi8-r',
'koi8-u',
'macintosh',
'x-mac-cyrillic',
'iso-8859-2',
'iso-8859-3',
'iso-8859-4',
'iso-8859-5',
'iso-8859-6',
'iso-8859-7',
'iso-8859-8',
'iso-8859-8-i',
'iso-8859-10',
'iso-8859-13',
'iso-8859-14',
'iso-8859-15',
'iso-8859-16',
'windows-874',
'windows-1250',
'windows-1251',
'windows-1252',
'windows-1253',
'windows-1254',
'windows-1255',
'windows-1256',
'windows-1257',
'windows-1258',
'x-user-defined', // Has to be last, special case
], (e, i) => [e, i])));

const isSinglebyteEncoding = (enc) => encodingsSinglebyte.has(enc);

function createSinglebyteDecoder(encoding, fatal) {
const key = encodingsSinglebyte.get(encoding);
if (key === undefined) throw new ERR_ENCODING_NOT_SUPPORTED(encoding);
return (buf) => decodeSingleByte(buf, key, fatal);
}

const encodings = new SafeMap([
['unicode-1-1-utf-8', 'utf-8'],
['unicode11utf8', 'utf-8'],
Expand Down Expand Up @@ -462,7 +504,7 @@ function makeTextDecoderICU() {
validateDecoder(this);
validateObject(options, 'options', kValidateObjectAllowObjectsAndNull);

if (this[kMethod]) return this[kMethod](parseInput(input));
if (this[kMethod]) return this[kMethod](input);

this[kUTF8FastPath] &&= !(options?.stream);

Expand Down Expand Up @@ -532,11 +574,12 @@ function makeTextDecoderJS() {

decode(input = empty, options = kEmptyObject) {
validateDecoder(this);
input = parseInput(input);
validateObject(options, 'options', kValidateObjectAllowObjectsAndNull);

if (this[kMethod]) return this[kMethod](input);

input = parseInput(input);

if (this[kFlags] & CONVERTER_FLAGS_FLUSH) {
this[kBOMSeen] = false;
}
Expand Down
155 changes: 0 additions & 155 deletions lib/internal/encoding/single-byte.js
View file
Open in desktop

This file was deleted.

2 changes: 2 additions & 0 deletions node.gyp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
'src/debug_utils.cc',
'src/embedded_data.cc',
'src/encoding_binding.cc',
'src/encoding_singlebyte.cc',
'src/env.cc',
'src/fs_event_wrap.cc',
'src/handle_wrap.cc',
Expand Down Expand Up @@ -219,6 +220,7 @@
'src/debug_utils-inl.h',
'src/embedded_data.h',
'src/encoding_binding.h',
'src/encoding_singlebyte.h',
'src/env_properties.h',
'src/env.h',
'src/env-inl.h',
Expand Down
70 changes: 70 additions & 0 deletions src/encoding_binding.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "encoding_binding.h"
#include "ada.h"
#include "encoding_singlebyte.h"
#include "env-inl.h"
#include "node_errors.h"
#include "node_external_reference.h"
Expand Down Expand Up @@ -391,6 +392,73 @@ void BindingData::DecodeUTF8(const FunctionCallbackInfo<Value>& args) {
}
}

void BindingData::DecodeSingleByte(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);

CHECK_GE(args.Length(), 2);
Isolate* isolate = env->isolate();

if (!(args[0]->IsArrayBuffer() || args[0]->IsSharedArrayBuffer() ||
args[0]->IsArrayBufferView())) {
return node::THROW_ERR_INVALID_ARG_TYPE(
isolate,
"The \"input\" argument must be an instance of SharedArrayBuffer, "
"ArrayBuffer or ArrayBufferView.");
}

CHECK(args[1]->IsInt32());
const int encoding = args[1].As<v8::Int32>()->Value();
CHECK(encoding >= 0 && encoding <= kXUserDefined);

ArrayBufferViewContents<uint8_t> buffer(args[0]);
const uint8_t* data = buffer.data();
size_t length = buffer.length();

if (length == 0) return args.GetReturnValue().SetEmptyString();

const char* dataChar = reinterpret_cast<const char*>(data);
if (!simdutf::validate_ascii_with_errors(dataChar, length).error) {
Local<Value> ret;
if (StringBytes::Encode(isolate, dataChar, length, LATIN1).ToLocal(&ret)) {
args.GetReturnValue().Set(ret);
}
return;
}

if (length > static_cast<size_t>(v8::String::kMaxLength)) {
isolate->ThrowException(ERR_STRING_TOO_LONG(isolate));
return;
}

uint16_t* dst = node::UncheckedMalloc<uint16_t>(length);
if (dst == nullptr) return node::THROW_ERR_MEMORY_ALLOCATION_FAILED(isolate);

if (encoding == kXUserDefined) {
// x-user-defined
for (size_t i = 0; i < length; i++) {
dst[i] = data[i] >= 0x80 ? data[i] + 0xf700 : data[i];
}
} else {
bool has_fatal = args[2]->IsTrue();

const uint16_t* table = tSingleByteEncodings[encoding];
for (size_t i = 0; i < length; i++) dst[i] = table[data[i]];

const char16_t* dst16 = reinterpret_cast<char16_t*>(dst);
if (has_fatal && fSingleByteEncodings[encoding] &&
simdutf::find(dst16, dst16 + length, 0xfffd) != dst16 + length) {
free(dst);
return node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(
isolate, "The encoded data was not valid for this encoding");
}
}

Local<Value> ret;
if (StringBytes::Raw(isolate, dst, length).ToLocal(&ret)) {
args.GetReturnValue().Set(ret);
}
}

void BindingData::ToASCII(const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
CHECK_GE(args.Length(), 1);
Expand Down Expand Up @@ -423,6 +491,7 @@ void BindingData::CreatePerIsolateProperties(IsolateData* isolate_data,
SetMethod(isolate, target, "encodeInto", EncodeInto);
SetMethodNoSideEffect(isolate, target, "encodeUtf8String", EncodeUtf8String);
SetMethodNoSideEffect(isolate, target, "decodeUTF8", DecodeUTF8);
SetMethodNoSideEffect(isolate, target, "decodeSingleByte", DecodeSingleByte);
SetMethodNoSideEffect(isolate, target, "toASCII", ToASCII);
SetMethodNoSideEffect(isolate, target, "toUnicode", ToUnicode);
}
Expand All @@ -440,6 +509,7 @@ void BindingData::RegisterTimerExternalReferences(
registry->Register(EncodeInto);
registry->Register(EncodeUtf8String);
registry->Register(DecodeUTF8);
registry->Register(DecodeSingleByte);
registry->Register(ToASCII);
registry->Register(ToUnicode);
}
Expand Down
Loading
Loading