Change in dart/sdk[master]: [vm] Extend AppJIT capability of gen_snapshot.
34 views
Skip to first unread message
Ryan Macnak (Gerrit)
Sep 18, 2018, 7:25:32 PM9/18/18
to rev...@dartlang.org, vm-...@dartlang.org, Siva Annamalai, Stanislav Baranov, commi...@chromium.org
Example usage:
./tools/build.py runtime
./out/DebugX64/gen_snapshot --snapshot-kind=core \
--strong \
--reify-generic-functions \
--sync-async \
--vm_snapshot_data=core.vm_data \
--isolate_snapshot_data=core.iso_data \
./out/DebugX64/vm_platform_strong.dill
./pkg/vm/tool/gen_kernel --platform out/DebugX64/vm_platform_strong.dill --output app1.dill app1.dart
./out/DebugX64/gen_snapshot --snapshot-kind=app-jit \
--strong \
--reify-generic-functions \
--sync-async \
--load_vm_snapshot_data=core.vm_data \
--load_isolate_snapshot_data=core.iso_data \
--isolate_snapshot_data=app1.iso_data \
--isolate_snapshot_instructions=app1.iso_instr \
--load-compilation-trace=$HOME/trace.txt \
app1.dill
./pkg/vm/tool/gen_kernel --platform out/DebugX64/vm_platform_strong.dill --output app2.dill app2.dart
./out/DebugX64/gen_snapshot --snapshot-kind=app-jit \
--trace-reused-instructions \
--strong \
--reify-generic-functions \
--sync-async \
--load_vm_snapshot_data=core.vm_data \
--load_isolate_snapshot_data=core.iso_data \
--isolate_snapshot_data=app2.iso_data \
--isolate_snapshot_instructions=app2.iso_instr \
--load-compilation-trace=$HOME/trace.txt \
--reused_instructions=app1.iso_instr \
app2.dill
To view, visit change 75390. To unsubscribe, or for help writing mail filters, visit settings.
Ryan Macnak (Gerrit)
Sep 18, 2018, 7:25:33 PM9/18/18
to Siva Annamalai, Stanislav Baranov, rev...@dartlang.org, vm-...@dartlang.org
Ryan Macnak would like Siva Annamalai and Stanislav Baranov to review this change.
[vm] Extend AppJIT capability of gen_snapshot.
- Add generation of AppJIT snapshots (from a trace rather than a training run).
- Add generation of AppJIT snapshots under the constraint of not generating a new instructions segment.
- Remove generation of script snapshots (no longer used by Flutter).
Change-Id: I2a7cb4b7cd681fae5d33bc896107b3ea903f35d1
---
M runtime/bin/gen_snapshot.cc
M runtime/bin/snapshot_utils.cc
M runtime/include/dart_api.h
M runtime/vm/dart_api_impl.cc
M runtime/vm/hash_map.h
M runtime/vm/image_snapshot.cc
M runtime/vm/image_snapshot.h
M runtime/vm/object.cc
8 files changed, 222 insertions(+), 112 deletions(-)
diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc
index 1a93dd0..7ed8a0e 100644
--- a/runtime/bin/gen_snapshot.cc
+++ b/runtime/bin/gen_snapshot.cc
@@ -60,7 +60,7 @@
}
// The core snapshot to use when creating isolates. Normally NULL, but loaded
-// from a file when creating script snapshots.
+// from a file when creating AppJIT snapshots.
const uint8_t* isolate_snapshot_data = NULL;
const uint8_t* isolate_snapshot_instructions = NULL;
@@ -69,7 +69,7 @@
enum SnapshotKind {
kCore,
kCoreJIT,
- kScript,
+ kAppJIT,
kAppAOTBlobs,
kAppAOTAssembly,
kVMAOTAssembly,
@@ -97,7 +97,7 @@
static const char* kSnapshotKindNames[] = {
"core",
"core-jit",
- "script",
+ "app-jit",
"app-aot-blobs",
"app-aot-assembly",
"vm-aot-assembly",
@@ -105,12 +105,18 @@
};
#define STRING_OPTIONS_LIST(V) \
+ V(load_vm_snapshot_data, load_vm_snapshot_data_filename) \
+ V(load_vm_snapshot_instructions, load_vm_snapshot_instructions_filename) \
+ V(load_isolate_snapshot_data, load_isolate_snapshot_data_filename) \
+ V(load_isolate_snapshot_instructions, \
+ load_isolate_snapshot_instructions_filename) \
V(vm_snapshot_data, vm_snapshot_data_filename) \
V(vm_snapshot_instructions, vm_snapshot_instructions_filename) \
V(isolate_snapshot_data, isolate_snapshot_data_filename) \
V(isolate_snapshot_instructions, isolate_snapshot_instructions_filename) \
V(shared_data, shared_data_filename) \
V(shared_instructions, shared_instructions_filename) \
+ V(reused_instructions, reused_instructions_filename) \
V(assembly, assembly_filename) \
V(script_snapshot, script_snapshot_filename) \
V(dependencies, dependencies_filename) \
@@ -150,10 +156,6 @@
(snapshot_kind == kVMAOTAssembly);
}
-static bool SnapshotKindAllowedFromKernel() {
- return snapshot_kind != kScript;
-}
-
// clang-format off
static void PrintUsage() {
Log::PrintErr(
@@ -318,14 +320,15 @@
}
break;
}
- case kScript: {
- if ((vm_snapshot_data_filename == NULL) ||
+ case kAppJIT: {
+ if ((load_vm_snapshot_data_filename == NULL) ||
(isolate_snapshot_data_filename == NULL) ||
- (script_snapshot_filename == NULL) || (*script_name == NULL)) {
+ (isolate_snapshot_instructions_filename == NULL)) {
Log::PrintErr(
- "Building a script snapshot requires specifying input files for "
- "--vm_snapshot_data and --isolate_snapshot_data, an output file "
- "for --script_snapshot, and a Dart script.\n\n");
+ "Building an app JIT snapshot requires specifying input files for "
+ "--load_vm_snapshot_data and --load_vm_snapshot_instructions and "
+ " output files for --isolate_snapshot_data and "
+ "--isolate_snapshot_instructions.\n\n");
return -1;
}
break;
@@ -577,12 +580,13 @@
WriteDependenciesWithTarget(vm_snapshot_data_filename);
// WriteDependenciesWithTarget(isolate_snapshot_data_filename);
break;
- case kScript:
- WriteDependenciesWithTarget(script_snapshot_filename);
- break;
case kAppAOTAssembly:
WriteDependenciesWithTarget(assembly_filename);
break;
+ case kAppJIT:
+ WriteDependenciesWithTarget(isolate_snapshot_data_filename);
+ // WriteDependenciesWithTarget(isolate_snapshot_instructions_filename);
+ break;
case kCoreJIT:
case kAppAOTBlobs:
WriteDependenciesWithTarget(vm_snapshot_data_filename);
@@ -606,21 +610,6 @@
WritePath(target);
Write(": ");
- if (snapshot_kind == kScript) {
- if (vm_snapshot_data_filename != NULL) {
- WritePath(vm_snapshot_data_filename);
- }
- if (vm_snapshot_instructions_filename != NULL) {
- WritePath(vm_snapshot_instructions_filename);
- }
- if (isolate_snapshot_data_filename != NULL) {
- WritePath(isolate_snapshot_data_filename);
- }
- if (isolate_snapshot_instructions_filename != NULL) {
- WritePath(isolate_snapshot_instructions_filename);
- }
- }
-
for (intptr_t i = 0; i < dependencies_->length(); i++) {
WritePath(dependencies_->At(i));
}
@@ -1072,7 +1061,7 @@
static void LoadCompilationTrace() {
if ((load_compilation_trace_filename != NULL) &&
- (snapshot_kind == kCoreJIT)) {
+ ((snapshot_kind == kCoreJIT) || (snapshot_kind == kAppJIT))) {
uint8_t* buffer = NULL;
intptr_t size = 0;
ReadFile(load_compilation_trace_filename, &buffer, &size);
@@ -1112,6 +1101,30 @@
}
}
+static std::unique_ptr<MappedMemory> MapFile(const char* filename,
+ File::MapType type,
+ const uint8_t** buffer) {
+ File* file = File::Open(NULL, filename, File::kRead);
+ if (file == NULL) {
+ Log::PrintErr("Failed to open: %s\n", filename);
+ exit(kErrorExitCode);
+ }
+ RefCntReleaseScope<File> rs(file);
+ intptr_t length = file->Length();
+ if (length == 0) {
+ // Can't map an empty file.
+ *buffer = NULL;
+ return NULL;
+ }
+ MappedMemory* mapping = file->Map(type, 0, length);
+ if (mapping == NULL) {
+ Log::PrintErr("Failed to read: %s\n", filename);
+ exit(kErrorExitCode);
+ }
+ *buffer = reinterpret_cast<const uint8_t*>(mapping->address());
+ return std::unique_ptr<MappedMemory>(mapping);
+}
+
static void CreateAndWriteCoreJITSnapshot() {
ASSERT(snapshot_kind == kCoreJIT);
ASSERT(vm_snapshot_data_filename != NULL);
@@ -1151,18 +1164,37 @@
isolate_snapshot_instructions_size);
}
-static void CreateAndWriteScriptSnapshot() {
- ASSERT(snapshot_kind == kScript);
- ASSERT(script_snapshot_filename != NULL);
+static void CreateAndWriteAppJITSnapshot() {
+ ASSERT(snapshot_kind == kAppJIT);
+ ASSERT(isolate_snapshot_data_filename != NULL);
+ ASSERT(isolate_snapshot_instructions_filename != NULL);
- // First create a snapshot.
- uint8_t* buffer = NULL;
- intptr_t size = 0;
- Dart_Handle result = Dart_CreateScriptSnapshot(&buffer, &size);
+ const uint8_t* reused_instructions = NULL;
+ std::unique_ptr<MappedMemory> mapped_reused_instructions;
+ if (reused_instructions_filename != NULL) {
+ mapped_reused_instructions = MapFile(reused_instructions_filename,
+ File::kReadOnly, &reused_instructions);
+ }
+
+ Dart_Handle result;
+ uint8_t* isolate_snapshot_data_buffer = NULL;
+ intptr_t isolate_snapshot_data_size = 0;
+ uint8_t* isolate_snapshot_instructions_buffer = NULL;
+ intptr_t isolate_snapshot_instructions_size = 0;
+
+ result = Dart_CreateAppJITSnapshotAsBlobs(
+ &isolate_snapshot_data_buffer, &isolate_snapshot_data_size,
+ &isolate_snapshot_instructions_buffer,
+ &isolate_snapshot_instructions_size, reused_instructions);
CHECK_RESULT(result);
- // Now write it out to the specified file.
- WriteFile(script_snapshot_filename, buffer, size);
+ WriteFile(isolate_snapshot_data_filename, isolate_snapshot_data_buffer,
+ isolate_snapshot_data_size);
+ if (reused_instructions_filename == NULL) {
+ WriteFile(isolate_snapshot_instructions_filename,
+ isolate_snapshot_instructions_buffer,
+ isolate_snapshot_instructions_size);
+ }
}
static void StreamingWriteCallback(void* callback_data,
@@ -1177,30 +1209,6 @@
}
}
-static std::unique_ptr<MappedMemory> MapFile(const char* filename,
- File::MapType type,
- const uint8_t** buffer) {
- File* file = File::Open(NULL, filename, File::kRead);
- if (file == NULL) {
- Log::PrintErr("Failed to open: %s\n", filename);
- exit(kErrorExitCode);
- }
- RefCntReleaseScope<File> rs(file);
- intptr_t length = file->Length();
- if (length == 0) {
- // Can't map an empty file.
- *buffer = NULL;
- return NULL;
- }
- MappedMemory* mapping = file->Map(type, 0, length);
- if (mapping == NULL) {
- Log::PrintErr("Failed to read: %s\n", filename);
- exit(kErrorExitCode);
- }
- *buffer = reinterpret_cast<const uint8_t*>(mapping->address());
- return std::unique_ptr<MappedMemory>(mapping);
-}
-
static void CreateAndWritePrecompiledSnapshot(
Dart_QualifiedFunctionName* standalone_entry_points) {
ASSERT(IsSnapshottingForPrecompilation());
@@ -1354,8 +1362,6 @@
static int GenerateSnapshotFromKernel(const uint8_t* kernel_buffer,
intptr_t kernel_buffer_size) {
- ASSERT(SnapshotKindAllowedFromKernel());
-
char* error = NULL;
IsolateData* isolate_data = new IsolateData(NULL, commandline_package_root,
commandline_packages_file, NULL);
@@ -1373,12 +1379,19 @@
isolate_flags.entry_points = entry_points;
}
- // We need to capture the vmservice library in the core snapshot, so load it
- // in the main isolate as well.
- isolate_flags.load_vmservice_library = true;
- Dart_Isolate isolate = Dart_CreateIsolateFromKernel(
- NULL, NULL, kernel_buffer, kernel_buffer_size, &isolate_flags,
- isolate_data, &error);
+ Dart_Isolate isolate;
+ if (isolate_snapshot_data == NULL) {
+ // We need to capture the vmservice library in the core snapshot, so load it
+ // in the main isolate as well.
+ isolate_flags.load_vmservice_library = true;
+ isolate = Dart_CreateIsolateFromKernel(NULL, NULL, kernel_buffer,
+ kernel_buffer_size, &isolate_flags,
+ isolate_data, &error);
+ } else {
+ isolate = Dart_CreateIsolate(NULL, NULL, isolate_snapshot_data,
+ isolate_snapshot_instructions, NULL, NULL,
+ &isolate_flags, isolate_data, &error);
+ }
if (isolate == NULL) {
delete isolate_data;
Log::PrintErr("%s\n", error);
@@ -1433,6 +1446,10 @@
LoadCompilationTrace();
CreateAndWriteCoreJITSnapshot();
break;
+ case kAppJIT:
+ LoadCompilationTrace();
+ CreateAndWriteAppJITSnapshot();
+ break;
default:
UNREACHABLE();
}
@@ -1484,13 +1501,6 @@
dfe.ReadScript(app_script_name, &kernel_buffer, &kernel_buffer_size);
}
if (kernel_buffer != NULL) {
- if (!SnapshotKindAllowedFromKernel()) {
- // TODO(sivachandra): Add check for the kernel program format (incremental
- // vs batch).
- Log::PrintErr(
- "Can only generate core or aot snapshots from a kernel file.\n");
- return kErrorExitCode;
- }
if (dependencies_filename != NULL) {
Log::PrintErr("Depfiles are not supported in Dart 2.\n");
return kErrorExitCode;
@@ -1517,7 +1527,7 @@
if (IsSnapshottingForPrecompilation()) {
vm_options.AddArgument("--precompilation");
}
- if (snapshot_kind == kCoreJIT) {
+ if (snapshot_kind == kCoreJIT || snapshot_kind == kAppJIT) {
vm_options.AddArgument("--fields_may_be_reset");
vm_options.AddArgument("--link_natives_lazily");
#if !defined(PRODUCT)
@@ -1558,26 +1568,25 @@
std::unique_ptr<MappedMemory> mapped_vm_snapshot_instructions;
std::unique_ptr<MappedMemory> mapped_isolate_snapshot_data;
std::unique_ptr<MappedMemory> mapped_isolate_snapshot_instructions;
- if (snapshot_kind == kScript) {
+ if (load_vm_snapshot_data_filename != NULL) {
mapped_vm_snapshot_data =
- MapFile(vm_snapshot_data_filename, File::kReadOnly,
+ MapFile(load_vm_snapshot_data_filename, File::kReadOnly,
&init_params.vm_snapshot_data);
-
- if (vm_snapshot_instructions_filename != NULL) {
- mapped_vm_snapshot_instructions =
- MapFile(vm_snapshot_instructions_filename, File::kReadExecute,
- &init_params.vm_snapshot_instructions);
- }
-
+ }
+ if (load_vm_snapshot_instructions_filename != NULL) {
+ mapped_vm_snapshot_instructions =
+ MapFile(load_vm_snapshot_instructions_filename, File::kReadExecute,
+ &init_params.vm_snapshot_instructions);
+ }
+ if (load_isolate_snapshot_data_filename) {
mapped_isolate_snapshot_data =
- MapFile(isolate_snapshot_data_filename, File::kReadOnly,
+ MapFile(load_isolate_snapshot_data_filename, File::kReadOnly,
&isolate_snapshot_data);
-
- if (isolate_snapshot_instructions_filename != NULL) {
- mapped_isolate_snapshot_instructions =
- MapFile(isolate_snapshot_instructions_filename, File::kReadExecute,
- &isolate_snapshot_instructions);
- }
+ }
+ if (load_isolate_snapshot_instructions_filename != NULL) {
+ mapped_isolate_snapshot_instructions =
+ MapFile(load_isolate_snapshot_instructions_filename, File::kReadExecute,
+ &isolate_snapshot_instructions);
}
error = Dart_Initialize(&init_params);
@@ -1713,8 +1722,8 @@
case kCoreJIT:
CreateAndWriteCoreJITSnapshot();
break;
- case kScript:
- CreateAndWriteScriptSnapshot();
+ case kAppJIT:
+ CreateAndWriteAppJITSnapshot();
break;
case kAppAOTBlobs:
case kAppAOTAssembly:
@@ -1743,6 +1752,9 @@
case kCoreJIT:
CreateAndWriteCoreJITSnapshot();
break;
+ case kAppJIT:
+ CreateAndWriteAppJITSnapshot();
+ break;
default:
UNREACHABLE();
break;
diff --git a/runtime/bin/snapshot_utils.cc b/runtime/bin/snapshot_utils.cc
index 2ff01c2..99ee0ec 100644
--- a/runtime/bin/snapshot_utils.cc
+++ b/runtime/bin/snapshot_utils.cc
@@ -368,7 +368,7 @@
void Snapshot::GenerateAppJIT(const char* snapshot_filename) {
#if defined(TARGET_ARCH_IA32)
- // Snapshots with code are not supported on IA32 or DBC.
+ // Snapshots with code are not supported on IA32.
uint8_t* isolate_buffer = NULL;
intptr_t isolate_size = 0;
@@ -387,7 +387,7 @@
intptr_t isolate_instructions_size = 0;
Dart_Handle result = Dart_CreateAppJITSnapshotAsBlobs(
&isolate_data_buffer, &isolate_data_size, &isolate_instructions_buffer,
- &isolate_instructions_size);
+ &isolate_instructions_size, NULL);
if (Dart_IsError(result)) {
ErrorExit(kErrorExitCode, "%s\n", Dart_GetError(result));
}
diff --git a/runtime/include/dart_api.h b/runtime/include/dart_api.h
index c1c02df..518259a 100644
--- a/runtime/include/dart_api.h
+++ b/runtime/include/dart_api.h
@@ -3471,7 +3471,8 @@
Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer,
intptr_t* isolate_snapshot_data_size,
uint8_t** isolate_snapshot_instructions_buffer,
- intptr_t* isolate_snapshot_instructions_size);
+ intptr_t* isolate_snapshot_instructions_size,
+ const uint8_t* reused_instructions);
/**
* Like Dart_CreateAppJITSnapshotAsBlobs, but also creates a new VM snapshot.
diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc
index 717daf3..2594354 100644
--- a/runtime/vm/dart_api_impl.cc
+++ b/runtime/vm/dart_api_impl.cc
@@ -6598,7 +6598,8 @@
Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer,
intptr_t* isolate_snapshot_data_size,
uint8_t** isolate_snapshot_instructions_buffer,
- intptr_t* isolate_snapshot_instructions_size) {
+ intptr_t* isolate_snapshot_instructions_size,
+ const uint8_t* reused_instructions) {
#if defined(TARGET_ARCH_IA32)
return Api::NewError("Snapshots with code are not supported on IA32.");
#elif defined(DART_PRECOMPILED_RUNTIME)
@@ -6622,6 +6623,9 @@
}
BackgroundCompiler::Stop(I);
+ if (reused_instructions) {
+ DropCodeWithoutReusableInstructions(reused_instructions);
+ }
ProgramVisitor::Dedup();
Symbols::Compact(I);
@@ -6629,7 +6633,7 @@
"WriteAppJITSnapshot"));
BlobImageWriter isolate_image_writer(isolate_snapshot_instructions_buffer,
ApiReallocate, 2 * MB /* initial_size */,
- NULL, NULL);
+ NULL, NULL, reused_instructions);
FullSnapshotWriter writer(Snapshot::kFullJIT, NULL,
isolate_snapshot_data_buffer, ApiReallocate, NULL,
&isolate_image_writer);
@@ -6639,6 +6643,10 @@
*isolate_snapshot_instructions_size =
isolate_image_writer.InstructionsBlobSize();
+ if (reused_instructions) {
+ *isolate_snapshot_instructions_buffer = NULL;
+ }
+
return Api::Success();
#endif
}
diff --git a/runtime/vm/hash_map.h b/runtime/vm/hash_map.h
index fb6ff15..95194aa 100644
--- a/runtime/vm/hash_map.h
+++ b/runtime/vm/hash_map.h
@@ -43,6 +43,7 @@
return Lookup(key) != NULL;
}
+ intptr_t Size() const { return count_; }
bool IsEmpty() const { return count_ == 0; }
virtual void Clear() {
diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc
index 28dda6a..ca3045a 100644
--- a/runtime/vm/image_snapshot.cc
+++ b/runtime/vm/image_snapshot.cc
@@ -12,6 +12,7 @@
#include "vm/heap/heap.h"
#include "vm/json_writer.h"
#include "vm/object.h"
+#include "vm/program_visitor.h"
#include "vm/stub_code.h"
#include "vm/timeline.h"
#include "vm/type_testing_stubs.h"
@@ -30,6 +31,11 @@
"Print sizes of all instruction objects to the given file");
#endif
+DEFINE_FLAG(bool,
+ trace_reused_instructions,
+ false,
+ "Print code that lacks reusable instructions");
+
intptr_t ObjectOffsetTrait::Hashcode(Key key) {
RawObject* obj = key;
ASSERT(!obj->IsSmi());
@@ -72,11 +78,13 @@
}
ImageWriter::ImageWriter(const void* shared_objects,
- const void* shared_instructions)
+ const void* shared_instructions,
+ const void* reused_instructions)
: next_data_offset_(0), next_text_offset_(0), objects_(), instructions_() {
ResetOffsets();
SetupShared(&shared_objects_, shared_objects);
SetupShared(&shared_instructions_, shared_instructions);
+ SetupShared(&reuse_instructions_, reused_instructions);
}
void ImageWriter::SetupShared(ObjectOffsetMap* map, const void* shared_image) {
@@ -100,6 +108,15 @@
int32_t ImageWriter::GetTextOffsetFor(RawInstructions* instructions,
RawCode* code) {
+ if (!reuse_instructions_.IsEmpty()) {
+ ObjectOffsetPair* pair = reuse_instructions_.Lookup(instructions);
+ if (pair == NULL) {
+ // Code should have been removed by DropCodeWithoutReusableInstructions.
+ FATAL("Expected instructions to reuse\n");
+ }
+ return pair->offset;
+ }
+
ObjectOffsetPair* pair = shared_instructions_.Lookup(instructions);
if (pair != NULL) {
// Negative offsets tell the reader the offset is w/r/t the shared
@@ -280,7 +297,7 @@
void* callback_data,
const void* shared_objects,
const void* shared_instructions)
- : ImageWriter(shared_objects, shared_instructions),
+ : ImageWriter(shared_objects, shared_instructions, NULL),
assembly_stream_(512 * KB, callback, callback_data),
dwarf_(NULL) {
#if defined(DART_PRECOMPILER)
@@ -531,8 +548,9 @@
ReAlloc alloc,
intptr_t initial_size,
const void* shared_objects,
- const void* shared_instructions)
- : ImageWriter(shared_objects, shared_instructions),
+ const void* shared_instructions,
+ const void* reused_instructions)
+ : ImageWriter(shared_objects, shared_instructions, reused_instructions),
instructions_blob_stream_(instructions_blob_buffer, alloc, initial_size) {
}
@@ -644,4 +662,67 @@
return result;
}
+void DropCodeWithoutReusableInstructions(const void* reused_instructions) {
+ class DropCodeVisitor : public FunctionVisitor, public ClassVisitor {
+ public:
+ explicit DropCodeVisitor(const void* reused_instructions)
+ : code_(Code::Handle()), instructions_(Instructions::Handle()) {
+ ImageWriter::SetupShared(&reused_instructions_, reused_instructions);
+ if (FLAG_trace_reused_instructions) {
+ OS::PrintErr("%" Pd " reusables instructions\n",
+ reused_instructions_.Size());
+ }
+ }
+
+ void Visit(const Class& cls) {
+ code_ = cls.allocation_stub();
+ if (!code_.IsNull() && !IsAvailable(code_)) {
+ if (FLAG_trace_reused_instructions) {
+ OS::PrintErr("No reusable instructions for %s\n", cls.ToCString());
+ }
+ cls.DisableAllocationStub();
+ }
+ }
+
+ void Visit(const Function& func) {
+ if (func.HasCode()) {
+ code_ = func.CurrentCode();
+ if (!IsAvailable(code_)) {
+ if (FLAG_trace_reused_instructions) {
+ OS::PrintErr("No reusable instructions for %s\n", func.ToCString());
+ }
+ func.ClearCode();
+ func.ClearICDataArray();
+ return;
+ }
+ }
+ code_ = func.unoptimized_code();
+ if (!code_.IsNull() && !IsAvailable(code_)) {
+ if (FLAG_trace_reused_instructions) {
+ OS::PrintErr("No reusable instructions for %s\n", func.ToCString());
+ }
+ func.ClearCode();
+ func.ClearICDataArray();
+ return;
+ }
+ }
+
+ private:
+ bool IsAvailable(const Code& code) {
+ ObjectOffsetPair* pair = reused_instructions_.Lookup(code.instructions());
+ return pair != NULL;
+ }
+
+ ObjectOffsetMap reused_instructions_;
+ Code& code_;
+ Instructions& instructions_;
+
+ DISALLOW_COPY_AND_ASSIGN(DropCodeVisitor);
+ };
+
+ DropCodeVisitor visitor(reused_instructions);
+ ProgramVisitor::VisitClasses(&visitor);
+ ProgramVisitor::VisitFunctions(&visitor);
+}
+
} // namespace dart
diff --git a/runtime/vm/image_snapshot.h b/runtime/vm/image_snapshot.h
index a4af7e4..b5ba9dc 100644
--- a/runtime/vm/image_snapshot.h
+++ b/runtime/vm/image_snapshot.h
@@ -96,10 +96,12 @@
class ImageWriter : public ValueObject {
public:
- ImageWriter(const void* shared_objects, const void* shared_instructions);
+ ImageWriter(const void* shared_objects,
+ const void* shared_instructions,
+ const void* reused_instructions);
virtual ~ImageWriter() {}
- void SetupShared(ObjectOffsetMap* map, const void* shared_image);
+ static void SetupShared(ObjectOffsetMap* map, const void* shared_image);
void ResetOffsets() {
next_data_offset_ = Image::kHeaderSize;
next_text_offset_ = Image::kHeaderSize;
@@ -156,6 +158,7 @@
GrowableArray<InstructionsData> instructions_;
ObjectOffsetMap shared_objects_;
ObjectOffsetMap shared_instructions_;
+ ObjectOffsetMap reuse_instructions_;
private:
DISALLOW_COPY_AND_ASSIGN(ImageWriter);
@@ -196,7 +199,8 @@
ReAlloc alloc,
intptr_t initial_size,
const void* shared_objects,
- const void* shared_instructions);
+ const void* shared_instructions,
+ const void* reused_instructions = NULL);
virtual void WriteText(WriteStream* clustered_stream, bool vm);
@@ -210,6 +214,8 @@
DISALLOW_COPY_AND_ASSIGN(BlobImageWriter);
};
+void DropCodeWithoutReusableInstructions(const void* reused_instructions);
+
} // namespace dart
#endif // RUNTIME_VM_IMAGE_SNAPSHOT_H_
diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc
index 54d9bd5..f91e438 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -1275,7 +1275,8 @@
"Object::Init");)
#if defined(DART_NO_SNAPSHOT)
- bool bootstrapping = Dart::vm_snapshot_kind() == Snapshot::kNone;
+ bool bootstrapping =
+ (Dart::vm_snapshot_kind() == Snapshot::kNone) || is_kernel;
#elif defined(DART_PRECOMPILED_RUNTIME)
bool bootstrapping = false;
#else
To view, visit change 75390. To unsubscribe, or for help writing mail filters, visit settings.
Vyacheslav Egorov (Gerrit)
Sep 18, 2018, 7:49:12 PM9/18/18
to Ryan Macnak, rev...@dartlang.org, vm-...@dartlang.org, Siva Annamalai, Stanislav Baranov, commi...@chromium.org
What is our long term plan with respect to gen_snapshot and dart(_bootstrap) binaries?
Do we really need both long term?
Ryan Macnak (Gerrit)
Sep 24, 2018, 6:06:10 PM9/24/18
to rev...@dartlang.org, vm-...@dartlang.org, Vyacheslav Egorov, Siva Annamalai, Stanislav Baranov, commi...@chromium.org
Patch Set 4:
What is our long term plan with respect to gen_snapshot and dart(_bootstrap) binaries?
Do we really need both long term?
Given everyone except the standalone embedder is using gen_snapshot, we should probably get rid of dart_bootstrap. I think the main issue was related to the various ways entry points were specified, but now that we only use the pragmas, it should be straightforward to switch over the standalone tests to gen_snapshot.
Stanislav Baranov (Gerrit)
Sep 25, 2018, 12:13:05 PM9/25/18
to Ryan Macnak, rev...@dartlang.org, vm-...@dartlang.org, Vyacheslav Egorov, Siva Annamalai, commi...@chromium.org
Patch set 4:Code-Review +1
Stanislav Baranov (Gerrit)
Sep 25, 2018, 7:38:19 PM9/25/18
to Ryan Macnak, rev...@dartlang.org, vm-...@dartlang.org, Vyacheslav Egorov, Siva Annamalai, commi...@chromium.org
Patch Set 4: Code-Review+1
Nit: Don't require --isolate_snapshot_instructions if --reused_instructions is specified.
Ryan Macnak (Gerrit)
Sep 25, 2018, 7:44:16 PM9/25/18
to rev...@dartlang.org, vm-...@dartlang.org, Stanislav Baranov, Vyacheslav Egorov, Siva Annamalai, commi...@chromium.org
Patch Set 4:
Patch Set 4: Code-Review+1
Nit: Don't require --isolate_snapshot_instructions if --reused_instructions is specified.
Done.
Patch set 5:Commit-Queue +2
commit-bot@chromium.org (Gerrit)
Sep 25, 2018, 7:44:20 PM9/25/18
to Ryan Macnak, rev...@dartlang.org, vm-...@dartlang.org, Stanislav Baranov, Vyacheslav Egorov, Siva Annamalai
CQ is trying the patch.
Note: The patchset sent to CQ was uploaded after this CL was approved.
"don't require unused instructions output argument" https://dart-review.googlesource.com/c/75390/5
Follow status at: https://chromium-cq-status.appspot.com/v2/patch-status/dart-review.googlesource.com/75390/5
Bot data: {"action": "start", "triggered_at": "2018-09-25T23:44:15.0Z", "cq_cfg_revision": "8ff692acf4ad71ec740855f9a2709c5f85517c99", "revision": "5eae1d5ddce5f4abcf888ed33adfd100996092ae"}
commit-bot@chromium.org (Gerrit)
Sep 25, 2018, 7:44:31 PM9/25/18
to Ryan Macnak, rev...@dartlang.org, vm-...@dartlang.org, Stanislav Baranov, Vyacheslav Egorov, Siva Annamalai
commi...@chromium.org merged this change.
Approvals:
Stanislav Baranov: Looks good to me, approved
Ryan Macnak: Commit
[vm] Extend AppJIT capability of gen_snapshot.
- Add generation of AppJIT snapshots (from a trace rather than a training run).
- Add generation of AppJIT snapshots under the constraint of not generating a new instructions segment.
- Remove generation of script snapshots (no longer used by Flutter).
Change-Id: I2a7cb4b7cd681fae5d33bc896107b3ea903f35d1
Reviewed-on: https://dart-review.googlesource.com/75390
Commit-Queue: Ryan Macnak <rma...@google.com>
Reviewed-by: Stanislav Baranov <sbar...@google.com>
---
M runtime/bin/gen_snapshot.cc
M runtime/bin/snapshot_utils.cc
M runtime/include/dart_api.h
M runtime/vm/dart_api_impl.cc
M runtime/vm/hash_map.h
M runtime/vm/image_snapshot.cc
M runtime/vm/image_snapshot.h
M runtime/vm/object.cc
8 files changed, 225 insertions(+), 112 deletions(-)
diff --git a/runtime/bin/gen_snapshot.cc b/runtime/bin/gen_snapshot.cc
index 690cf4d..3f6a71f 100644
--- a/runtime/bin/gen_snapshot.cc
+++ b/runtime/bin/gen_snapshot.cc
@@ -60,7 +60,7 @@
}
// The core snapshot to use when creating isolates. Normally NULL, but loaded
-// from a file when creating script snapshots.
+// from a file when creating AppJIT snapshots.
const uint8_t* isolate_snapshot_data = NULL;
const uint8_t* isolate_snapshot_instructions = NULL;
@@ -69,7 +69,7 @@
enum SnapshotKind {
kCore,
kCoreJIT,
- kScript,
+ kAppJIT,
kAppAOTBlobs,
kAppAOTAssembly,
kVMAOTAssembly,
@@ -93,7 +93,7 @@
static const char* kSnapshotKindNames[] = {
"core",
"core-jit",
- "script",
+ "app-jit",
"app-aot-blobs",
"app-aot-assembly",
"vm-aot-assembly",
@@ -101,12 +101,18 @@@@ -305,14 +307,17 @@
}
break;
}
- case kScript: {
- if ((vm_snapshot_data_filename == NULL) ||
+ case kAppJIT: {
+ if ((load_vm_snapshot_data_filename == NULL) ||
(isolate_snapshot_data_filename == NULL) ||
- (script_snapshot_filename == NULL) || (*script_name == NULL)) {
+ ((isolate_snapshot_instructions_filename == NULL) &&
+ (reused_instructions_filename == NULL))) {
Log::PrintErr(
- "Building a script snapshot requires specifying input files for "
- "--vm_snapshot_data and --isolate_snapshot_data, an output file "
- "for --script_snapshot, and a Dart script.\n\n");
+ "Building an app JIT snapshot requires specifying input files for "
+ "--load_vm_snapshot_data and --load_vm_snapshot_instructions, an "
+ " output file for --isolate_snapshot_data, and either an output "
+ "file for --isolate_snapshot_instructions or an input file for "
+ "--reused_instructions.\n\n");
return -1;
}
break;
@@ -564,12 +569,13 @@
WriteDependenciesWithTarget(vm_snapshot_data_filename);
// WriteDependenciesWithTarget(isolate_snapshot_data_filename);
break;
- case kScript:
- WriteDependenciesWithTarget(script_snapshot_filename);
- break;
case kAppAOTAssembly:
WriteDependenciesWithTarget(assembly_filename);
break;
+ case kAppJIT:
+ WriteDependenciesWithTarget(isolate_snapshot_data_filename);
+ // WriteDependenciesWithTarget(isolate_snapshot_instructions_filename);
+ break;
case kCoreJIT:
case kAppAOTBlobs:
WriteDependenciesWithTarget(vm_snapshot_data_filename);
@@ -593,21 +599,6 @@
WritePath(target);
Write(": ");
- if (snapshot_kind == kScript) {
- if (vm_snapshot_data_filename != NULL) {
- WritePath(vm_snapshot_data_filename);
- }
- if (vm_snapshot_instructions_filename != NULL) {
- WritePath(vm_snapshot_instructions_filename);
- }
- if (isolate_snapshot_data_filename != NULL) {
- WritePath(isolate_snapshot_data_filename);
- }
- if (isolate_snapshot_instructions_filename != NULL) {
- WritePath(isolate_snapshot_instructions_filename);
- }
- }
-
for (intptr_t i = 0; i < dependencies_->length(); i++) {
WritePath(dependencies_->At(i));
}
@@ -765,7 +756,7 @@
static void LoadCompilationTrace() {
if ((load_compilation_trace_filename != NULL) &&
- (snapshot_kind == kCoreJIT)) {
+ ((snapshot_kind == kCoreJIT) || (snapshot_kind == kAppJIT))) {
uint8_t* buffer = NULL;
intptr_t size = 0;
ReadFile(load_compilation_trace_filename, &buffer, &size);
@@ -805,6 +796,30 @@@@ -844,18 +859,38 @@
isolate_snapshot_instructions_size);
}
-static void CreateAndWriteScriptSnapshot() {
- ASSERT(snapshot_kind == kScript);
- ASSERT(script_snapshot_filename != NULL);
+static void CreateAndWriteAppJITSnapshot() {
+ ASSERT(snapshot_kind == kAppJIT);
+ ASSERT(isolate_snapshot_data_filename != NULL);
+ ASSERT((isolate_snapshot_instructions_filename != NULL) ||
+ (reused_instructions_filename != NULL));
@@ -870,30 +905,6 @@
}
}
-static std::unique_ptr<MappedMemory> MapFile(const char* filename,
- File::MapType type,
- const uint8_t** buffer) {
- File* file = File::Open(NULL, filename, File::kRead);
- if (file == NULL) {
- Log::PrintErr("Failed to open: %s\n", filename);
- exit(kErrorExitCode);
- }
- RefCntReleaseScope<File> rs(file);
- intptr_t length = file->Length();
- if (length == 0) {
- // Can't map an empty file.
- *buffer = NULL;
- return NULL;
- }
- MappedMemory* mapping = file->Map(type, 0, length);
- if (mapping == NULL) {
- Log::PrintErr("Failed to read: %s\n", filename);
- exit(kErrorExitCode);
- }
- *buffer = reinterpret_cast<const uint8_t*>(mapping->address());
- return std::unique_ptr<MappedMemory>(mapping);
-}
-
static void CreateAndWritePrecompiledSnapshot() {
ASSERT(IsSnapshottingForPrecompilation());
Dart_Handle result;
@@ -1050,8 +1061,6 @@
static int GenerateSnapshotFromKernel(const uint8_t* kernel_buffer,
intptr_t kernel_buffer_size) {
- ASSERT(SnapshotKindAllowedFromKernel());
-
char* error = NULL;
IsolateData* isolate_data = new IsolateData(NULL, commandline_package_root,
commandline_packages_file, NULL);
@@ -1067,12 +1076,19 @@
isolate_flags.entry_points = no_entry_points;
}
- // We need to capture the vmservice library in the core snapshot, so load it
- // in the main isolate as well.
- isolate_flags.load_vmservice_library = true;
- Dart_Isolate isolate = Dart_CreateIsolateFromKernel(
- NULL, NULL, kernel_buffer, kernel_buffer_size, &isolate_flags,
- isolate_data, &error);
+ Dart_Isolate isolate;
+ if (isolate_snapshot_data == NULL) {
+ // We need to capture the vmservice library in the core snapshot, so load it
+ // in the main isolate as well.
+ isolate_flags.load_vmservice_library = true;
+ isolate = Dart_CreateIsolateFromKernel(NULL, NULL, kernel_buffer,
+ kernel_buffer_size, &isolate_flags,
+ isolate_data, &error);
+ } else {
+ isolate = Dart_CreateIsolate(NULL, NULL, isolate_snapshot_data,
+ isolate_snapshot_instructions, NULL, NULL,
+ &isolate_flags, isolate_data, &error);
+ }
if (isolate == NULL) {
delete isolate_data;
Log::PrintErr("%s\n", error);
@@ -1125,6 +1141,10 @@
LoadCompilationTrace();
CreateAndWriteCoreJITSnapshot();
break;
+ case kAppJIT:
+ LoadCompilationTrace();
+ CreateAndWriteAppJITSnapshot();
+ break;
default:
UNREACHABLE();
}
@@ -1172,13 +1192,6 @@
dfe.ReadScript(app_script_name, &kernel_buffer, &kernel_buffer_size);
}
if (kernel_buffer != NULL) {
- if (!SnapshotKindAllowedFromKernel()) {
- // TODO(sivachandra): Add check for the kernel program format (incremental
- // vs batch).
- Log::PrintErr(
- "Can only generate core or aot snapshots from a kernel file.\n");
- return kErrorExitCode;
- }
if (dependencies_filename != NULL) {
Log::PrintErr("Depfiles are not supported in Dart 2.\n");
return kErrorExitCode;
@@ -1205,7 +1218,7 @@
if (IsSnapshottingForPrecompilation()) {
vm_options.AddArgument("--precompilation");
}
- if (snapshot_kind == kCoreJIT) {
+ if (snapshot_kind == kCoreJIT || snapshot_kind == kAppJIT) {
vm_options.AddArgument("--fields_may_be_reset");
vm_options.AddArgument("--link_natives_lazily");
#if !defined(PRODUCT)
@@ -1246,26 +1259,25 @@@@ -1391,8 +1403,8 @@
case kCoreJIT:
CreateAndWriteCoreJITSnapshot();
break;
- case kScript:
- CreateAndWriteScriptSnapshot();
+ case kAppJIT:
+ CreateAndWriteAppJITSnapshot();
break;
case kAppAOTBlobs:
case kAppAOTAssembly:
@@ -1419,6 +1431,9 @@index a8df732..a41812c 100644
--- a/runtime/include/dart_api.h
+++ b/runtime/include/dart_api.h
@@ -3470,7 +3470,8 @@
Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer,
intptr_t* isolate_snapshot_data_size,
uint8_t** isolate_snapshot_instructions_buffer,
- intptr_t* isolate_snapshot_instructions_size);
+ intptr_t* isolate_snapshot_instructions_size,
+ const uint8_t* reused_instructions);
/**
* Like Dart_CreateAppJITSnapshotAsBlobs, but also creates a new VM snapshot.
diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc
index 7a079a0..1c4c4f5 100644
--- a/runtime/vm/dart_api_impl.cc
+++ b/runtime/vm/dart_api_impl.cc
@@ -6597,7 +6597,8 @@
Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer,
intptr_t* isolate_snapshot_data_size,
uint8_t** isolate_snapshot_instructions_buffer,
- intptr_t* isolate_snapshot_instructions_size) {
+ intptr_t* isolate_snapshot_instructions_size,
+ const uint8_t* reused_instructions) {
#if defined(TARGET_ARCH_IA32)
return Api::NewError("Snapshots with code are not supported on IA32.");
#elif defined(DART_PRECOMPILED_RUNTIME)
@@ -6621,6 +6622,9 @@
}
BackgroundCompiler::Stop(I);
+ if (reused_instructions) {
+ DropCodeWithoutReusableInstructions(reused_instructions);
+ }
ProgramVisitor::Dedup();
Symbols::Compact(I);
@@ -6628,7 +6632,7 @@
"WriteAppJITSnapshot"));
BlobImageWriter isolate_image_writer(isolate_snapshot_instructions_buffer,
ApiReallocate, 2 * MB /* initial_size */,
- NULL, NULL);
+ NULL, NULL, reused_instructions);
FullSnapshotWriter writer(Snapshot::kFullJIT, NULL,
isolate_snapshot_data_buffer, ApiReallocate, NULL,
&isolate_image_writer);
@@ -6638,6 +6642,10 @@
*isolate_snapshot_instructions_size =
isolate_image_writer.InstructionsBlobSize();
+ if (reused_instructions) {
+ *isolate_snapshot_instructions_buffer = NULL;
+ }
+
return Api::Success();
#endif
}
diff --git a/runtime/vm/hash_map.h b/runtime/vm/hash_map.h
index fb6ff15..95194aa 100644
--- a/runtime/vm/hash_map.h
+++ b/runtime/vm/hash_map.h
@@ -43,6 +43,7 @@
return Lookup(key) != NULL;
}
+ intptr_t Size() const { return count_; }
bool IsEmpty() const { return count_ == 0; }
virtual void Clear() {
diff --git a/runtime/vm/image_snapshot.cc b/runtime/vm/image_snapshot.cc
index 28dda6a..26dea3a 100644+ OS::PrintErr("%" Pd " reusable instructions\n",index 0a4719f..5a92e02 100644
--- a/runtime/vm/object.cc
+++ b/runtime/vm/object.cc
@@ -1276,7 +1276,8 @@
"Object::Init");)
#if defined(DART_NO_SNAPSHOT)
- bool bootstrapping = Dart::vm_snapshot_kind() == Snapshot::kNone;
+ bool bootstrapping =
+ (Dart::vm_snapshot_kind() == Snapshot::kNone) || is_kernel;
#elif defined(DART_PRECOMPILED_RUNTIME)
bool bootstrapping = false;
#else
To view, visit change 75390. To unsubscribe, or for help writing mail filters, visit settings.
Reply all
Reply to author
Forward
0 new messages