feat(host): vendor PyroWave + minimal Granite subset as crates/pyrowave-sys
Phase 0 of design/pyrowave-codec-plan.md — the opt-in wired-LAN ultra-low- latency codec. Vendored at upstream 509e4f88 (API 0.4.0, Granite 44362775, volk + vulkan-headers pins in PUNKTFUNK-VENDOR.txt), pruned to the 6.6 MB the standalone no-renderer build needs; scripts/vendor-pyrowave.sh reproduces the tree (a pin bump is protocol-affecting, plan §4.2). build.rs drives the wrapper CMakeLists (static archives incl. a static C-API lib upstream only ships shared) + bindgen over pyrowave.h; Linux and Windows only, empty stub elsewhere (Apple gets a native Metal port, §4.7). Offline-safe by construction: no network, no system lib, vendored Vulkan headers — same model as the opus dep (flatpak builder has no network). Phase-0 validation on .21 (RTX 5070 Ti, driver 610.43.03): - upstream pyrowave-c-test + interop test (incl. dmabuf/DRM-modifier Vulkan<->Vulkan) pass, from the pristine AND the pruned tree - GPU kernel times at ~1.6 bpp noise: encode/decode 0.090/0.042 ms @800p, 0.146/0.067 @1080p, 0.226/0.103 @1440p, 0.477/0.201 @4K — order of magnitude under NVENC's 1-2 ms retrieve, CBR lands within ~100 B of target - cargo test -p pyrowave-sys green (static link + API-version pin check) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
add_granite_internal_lib(granite-filesystem
|
||||
volatile_source.hpp
|
||||
filesystem.hpp filesystem.cpp
|
||||
asset_manager.cpp asset_manager.hpp)
|
||||
|
||||
if (WIN32)
|
||||
target_sources(granite-filesystem PRIVATE windows/os_filesystem.cpp windows/os_filesystem.hpp)
|
||||
target_include_directories(granite-filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/windows)
|
||||
elseif (ANDROID)
|
||||
target_sources(granite-filesystem PRIVATE linux/os_filesystem.cpp linux/os_filesystem.hpp)
|
||||
target_sources(granite-filesystem PRIVATE android/android.cpp android/android.hpp)
|
||||
target_include_directories(granite-filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux)
|
||||
target_include_directories(granite-filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/android)
|
||||
else()
|
||||
target_sources(granite-filesystem PRIVATE linux/os_filesystem.cpp linux/os_filesystem.hpp)
|
||||
target_include_directories(granite-filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux)
|
||||
endif()
|
||||
|
||||
target_include_directories(granite-filesystem PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_link_libraries(granite-filesystem PUBLIC granite-util granite-path granite-application-global PRIVATE granite-threading)
|
||||
|
||||
if (GRANITE_SHIPPING)
|
||||
target_compile_definitions(granite-filesystem PRIVATE GRANITE_SHIPPING)
|
||||
else()
|
||||
target_compile_definitions(granite-filesystem PRIVATE GRANITE_DEFAULT_BUILTIN_DIRECTORY=\"${CMAKE_CURRENT_SOURCE_DIR}/../assets\")
|
||||
target_compile_definitions(granite-filesystem PRIVATE GRANITE_DEFAULT_CACHE_DIRECTORY=\"${CMAKE_BINARY_DIR}/cache\")
|
||||
endif()
|
||||
@@ -0,0 +1,156 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "android.hpp"
|
||||
#include "path_utils.hpp"
|
||||
#include "logging.hpp"
|
||||
#include <algorithm>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
bool AssetFile::init(AAssetManager *mgr, const std::string &path, FileMode mode)
|
||||
{
|
||||
if (mode != FileMode::ReadOnly)
|
||||
{
|
||||
LOGE("Asset files must be opened read-only.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
asset = AAssetManager_open(mgr, path.c_str(), AASSET_MODE_BUFFER);
|
||||
if (!asset)
|
||||
return false;
|
||||
|
||||
size = AAsset_getLength64(asset);
|
||||
return true;
|
||||
}
|
||||
|
||||
FileHandle AssetFile::open(AAssetManager *mgr, const std::string &path, Granite::FileMode mode)
|
||||
{
|
||||
auto file = Util::make_handle<AssetFile>();
|
||||
if (!file->init(mgr, path, mode))
|
||||
file.reset();
|
||||
return file;
|
||||
}
|
||||
|
||||
FileMappingHandle AssetFile::map_subset(uint64_t offset, size_t range)
|
||||
{
|
||||
if (offset + range > size)
|
||||
return {};
|
||||
|
||||
auto *data = static_cast<uint8_t *>(const_cast<void *>(AAsset_getBuffer(asset)));
|
||||
if (!data)
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
reference_from_this(), offset,
|
||||
data + offset, range,
|
||||
0, range);
|
||||
}
|
||||
|
||||
uint64_t AssetFile::get_size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
FileMappingHandle AssetFile::map_write(size_t)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
void AssetFile::unmap(void *, size_t)
|
||||
{
|
||||
}
|
||||
|
||||
AssetFile::~AssetFile()
|
||||
{
|
||||
if (asset)
|
||||
AAsset_close(asset);
|
||||
}
|
||||
|
||||
AssetManagerFilesystem::AssetManagerFilesystem(const std::string &base_)
|
||||
: base(base_), mgr(global_asset_manager)
|
||||
{
|
||||
}
|
||||
|
||||
FileHandle AssetManagerFilesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
return AssetFile::open(mgr, Path::join(base, Path::canonicalize_path(path)), mode);
|
||||
}
|
||||
|
||||
int AssetManagerFilesystem::get_notification_fd() const
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
void AssetManagerFilesystem::poll_notifications()
|
||||
{
|
||||
}
|
||||
|
||||
void AssetManagerFilesystem::uninstall_notification(FileNotifyHandle)
|
||||
{
|
||||
}
|
||||
|
||||
FileNotifyHandle AssetManagerFilesystem::install_notification(const std::string &,
|
||||
std::function<void (const FileNotifyInfo &)>)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::vector<ListEntry> AssetManagerFilesystem::list(const std::string &path)
|
||||
{
|
||||
auto directory = Path::join(base, Path::canonicalize_path(path));
|
||||
auto *dir = AAssetManager_openDir(mgr, directory.c_str());
|
||||
|
||||
if (!dir)
|
||||
return {};
|
||||
|
||||
std::vector<ListEntry> entries;
|
||||
|
||||
const char *entry;
|
||||
while ((entry = AAssetDir_getNextFileName(dir)))
|
||||
{
|
||||
PathType type = PathType::File;
|
||||
entries.push_back({ entry, type });
|
||||
}
|
||||
|
||||
AAssetDir_close(dir);
|
||||
return entries;
|
||||
}
|
||||
|
||||
bool AssetManagerFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto resolved_path = Path::join(base, Path::canonicalize_path(path));
|
||||
|
||||
auto *asset = AAssetManager_open(mgr, resolved_path.c_str(), AASSET_MODE_UNKNOWN);
|
||||
if (!asset)
|
||||
return false;
|
||||
|
||||
stat.size = AAsset_getLength(asset);
|
||||
stat.type = PathType::File;
|
||||
stat.last_modified = 0;
|
||||
AAsset_close(asset);
|
||||
return true;
|
||||
}
|
||||
|
||||
AAssetManager *AssetManagerFilesystem::global_asset_manager;
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "../filesystem.hpp"
|
||||
#include <unordered_map>
|
||||
#include <android/asset_manager_jni.h>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
class AssetFile final : public File
|
||||
{
|
||||
public:
|
||||
static FileHandle open(AAssetManager *mgr, const std::string &path, FileMode mode);
|
||||
~AssetFile() override;
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t range) override;
|
||||
FileMappingHandle map_write(size_t size) override;
|
||||
void unmap(void *mapped, size_t mapped_range) override;
|
||||
uint64_t get_size() override;
|
||||
|
||||
private:
|
||||
bool init(AAssetManager *mgr, const std::string &path, FileMode mode);
|
||||
AAsset *asset = nullptr;
|
||||
size_t size = 0;
|
||||
};
|
||||
|
||||
class AssetManagerFilesystem : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
AssetManagerFilesystem(const std::string &base);
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
FileHandle open(const std::string &path, FileMode mode) override;
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void (const FileNotifyInfo &)> func) override;
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
void poll_notifications() override;
|
||||
int get_notification_fd() const override;
|
||||
|
||||
static AAssetManager *global_asset_manager;
|
||||
|
||||
private:
|
||||
std::string base;
|
||||
AAssetManager *mgr;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "asset_manager.hpp"
|
||||
#include "thread_group.hpp"
|
||||
#include <utility>
|
||||
#include <algorithm>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
AssetManager::AssetManager()
|
||||
{
|
||||
asset_bank.reserve(AssetID::MaxIDs);
|
||||
sorted_assets.reserve(AssetID::MaxIDs);
|
||||
signal = std::make_unique<TaskSignal>();
|
||||
for (uint64_t i = 0; i < timestamp; i++)
|
||||
signal->signal_increment();
|
||||
}
|
||||
|
||||
AssetManager::~AssetManager()
|
||||
{
|
||||
set_asset_instantiator_interface(nullptr);
|
||||
signal->wait_until_at_least(timestamp);
|
||||
for (uint32_t i = 0; i < id_count; i++)
|
||||
pool.free(asset_bank[i]);
|
||||
}
|
||||
|
||||
AssetID AssetManager::register_asset_nolock(FileHandle file, AssetClass asset_class, int prio)
|
||||
{
|
||||
auto *info = pool.allocate();
|
||||
info->handle = std::move(file);
|
||||
info->id.id = id_count;
|
||||
info->prio = prio;
|
||||
info->asset_class = asset_class;
|
||||
AssetID ret = info->id;
|
||||
asset_bank[id_count++] = info;
|
||||
if (iface)
|
||||
{
|
||||
iface->set_id_bounds(id_count);
|
||||
iface->set_asset_class(info->id, asset_class);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
void AssetInstantiatorInterface::set_asset_class(AssetID, AssetClass)
|
||||
{
|
||||
}
|
||||
|
||||
AssetID AssetManager::register_asset(FileHandle file, AssetClass asset_class, int prio)
|
||||
{
|
||||
std::lock_guard<std::mutex> holder{asset_bank_lock};
|
||||
return register_asset_nolock(std::move(file), asset_class, prio);
|
||||
}
|
||||
|
||||
AssetID AssetManager::register_asset(Filesystem &fs, const std::string &path, AssetClass asset_class, int prio)
|
||||
{
|
||||
std::lock_guard<std::mutex> holder{asset_bank_lock};
|
||||
|
||||
Util::Hasher h;
|
||||
h.string(path);
|
||||
if (auto *asset = file_to_assets.find(h.get()))
|
||||
return asset->id;
|
||||
|
||||
auto file = fs.open(path);
|
||||
if (!file)
|
||||
return {};
|
||||
|
||||
auto id = register_asset_nolock(std::move(file), asset_class, prio);
|
||||
asset_bank[id.id]->set_hash(h.get());
|
||||
file_to_assets.insert_replace(asset_bank[id.id]);
|
||||
return id;
|
||||
}
|
||||
|
||||
void AssetManager::update_cost(AssetID id, uint64_t cost)
|
||||
{
|
||||
std::lock_guard<std::mutex> holder{cost_update_lock};
|
||||
thread_cost_updates.push_back({ id, cost });
|
||||
}
|
||||
|
||||
void AssetManager::set_asset_instantiator_interface(AssetInstantiatorInterface *iface_)
|
||||
{
|
||||
if (iface)
|
||||
{
|
||||
signal->wait_until_at_least(timestamp);
|
||||
for (uint32_t id = 0; id < id_count; id++)
|
||||
iface->release_asset(AssetID{id});
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < id_count; i++)
|
||||
{
|
||||
auto *a = asset_bank[i];
|
||||
a->consumed = 0;
|
||||
a->pending_consumed = 0;
|
||||
a->last_used = 0;
|
||||
}
|
||||
total_consumed = 0;
|
||||
|
||||
iface = iface_;
|
||||
if (iface)
|
||||
{
|
||||
iface->set_id_bounds(id_count);
|
||||
for (uint32_t i = 0; i < id_count; i++)
|
||||
iface->set_asset_class(AssetID{i}, asset_bank[i]->asset_class);
|
||||
}
|
||||
}
|
||||
|
||||
void AssetManager::mark_used_asset(AssetID id)
|
||||
{
|
||||
lru_append.push(id);
|
||||
}
|
||||
|
||||
bool AssetManager::get_wants_mesh_assets() const
|
||||
{
|
||||
return wants_mesh_assets;
|
||||
}
|
||||
|
||||
void AssetManager::enable_mesh_assets()
|
||||
{
|
||||
wants_mesh_assets = true;
|
||||
}
|
||||
|
||||
void AssetManager::set_asset_budget(uint64_t cost)
|
||||
{
|
||||
transfer_budget = cost;
|
||||
}
|
||||
|
||||
void AssetManager::set_asset_budget_per_iteration(uint64_t cost)
|
||||
{
|
||||
transfer_budget_per_iteration = cost;
|
||||
}
|
||||
|
||||
bool AssetManager::set_asset_residency_priority(AssetID id, int prio)
|
||||
{
|
||||
std::lock_guard<std::mutex> holder{asset_bank_lock};
|
||||
if (id.id >= id_count)
|
||||
return false;
|
||||
asset_bank[id.id]->prio = prio;
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::adjust_update(const CostUpdate &update)
|
||||
{
|
||||
if (update.id.id < id_count)
|
||||
{
|
||||
auto *a = asset_bank[update.id.id];
|
||||
total_consumed += update.cost - (a->consumed + a->pending_consumed);
|
||||
a->consumed = update.cost;
|
||||
a->pending_consumed = 0;
|
||||
|
||||
// A recently paged in image shouldn't be paged out right away in a situation where we're thrashing,
|
||||
// that'd be very dumb.
|
||||
a->last_used = timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t AssetManager::get_current_total_consumed() const
|
||||
{
|
||||
return total_consumed;
|
||||
}
|
||||
|
||||
void AssetManager::update_costs_locked_assets()
|
||||
{
|
||||
{
|
||||
std::lock_guard<std::mutex> holder_cost{cost_update_lock};
|
||||
std::swap(cost_updates, thread_cost_updates);
|
||||
}
|
||||
|
||||
for (auto &update : cost_updates)
|
||||
adjust_update(update);
|
||||
cost_updates.clear();
|
||||
}
|
||||
|
||||
void AssetManager::update_lru_locked_assets()
|
||||
{
|
||||
lru_append.for_each_ranged([this](const AssetID *id, size_t count) {
|
||||
for (size_t i = 0; i < count; i++)
|
||||
if (id[i].id < id_count)
|
||||
asset_bank[id[i].id]->last_used = timestamp;
|
||||
});
|
||||
lru_append.clear();
|
||||
}
|
||||
|
||||
bool AssetManager::iterate_blocking(ThreadGroup &group, AssetID id)
|
||||
{
|
||||
if (!iface)
|
||||
return false;
|
||||
|
||||
std::lock_guard<std::mutex> holder{asset_bank_lock};
|
||||
update_costs_locked_assets();
|
||||
update_lru_locked_assets();
|
||||
|
||||
if (id.id >= id_count)
|
||||
return false;
|
||||
|
||||
auto *candidate = asset_bank[id.id];
|
||||
if (candidate->consumed != 0 || candidate->pending_consumed != 0)
|
||||
return true;
|
||||
|
||||
uint64_t estimate = iface->estimate_cost_asset(candidate->id, *candidate->handle);
|
||||
auto task = group.create_task();
|
||||
task->set_task_class(TaskClass::Background);
|
||||
task->set_fence_counter_signal(signal.get());
|
||||
task->set_desc("asset-manager-instantiate-single");
|
||||
iface->instantiate_asset(*this, task.get(), candidate->id, *candidate->handle);
|
||||
candidate->pending_consumed = estimate;
|
||||
candidate->last_used = timestamp;
|
||||
total_consumed += estimate;
|
||||
|
||||
// We cannot increment the timestamp here, remember this for later.
|
||||
// We hold a lock on the asset bank here, so this is fine even if called concurrently.
|
||||
blocking_signals++;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void AssetManager::iterate(ThreadGroup *group)
|
||||
{
|
||||
if (!iface)
|
||||
return;
|
||||
|
||||
timestamp += blocking_signals;
|
||||
blocking_signals = 0;
|
||||
|
||||
// If there is too much pending work in flight, skip.
|
||||
uint64_t current_count = signal->get_count();
|
||||
if (current_count + 3 < timestamp)
|
||||
{
|
||||
iface->latch_handles();
|
||||
LOGI("Asset manager skipping iteration due to too much pending work.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
TaskGroupHandle task;
|
||||
if (group)
|
||||
{
|
||||
task = group->create_task();
|
||||
task->set_desc("asset-manager-instantiate");
|
||||
task->set_fence_counter_signal(signal.get());
|
||||
task->set_task_class(TaskClass::Background);
|
||||
}
|
||||
else
|
||||
signal->signal_increment();
|
||||
|
||||
std::lock_guard<std::mutex> holder{asset_bank_lock};
|
||||
update_costs_locked_assets();
|
||||
update_lru_locked_assets();
|
||||
|
||||
memcpy(sorted_assets.data(), asset_bank.data(), id_count * sizeof(sorted_assets[0]));
|
||||
std::sort(sorted_assets.data(), sorted_assets.data() + id_count, [](const AssetInfo *a, const AssetInfo *b) -> bool {
|
||||
// High prios come first since they will be activated.
|
||||
// Then we sort by LRU.
|
||||
// High consumption should be moved last, so they are candidates to be paged out if we're over budget.
|
||||
// High pending consumption should be moved early since we don't want to page out resources that
|
||||
// are in the middle of being loaded anyway.
|
||||
// Finally, the ID is used as a tie breaker.
|
||||
|
||||
if (a->prio != b->prio)
|
||||
return a->prio > b->prio;
|
||||
else if (a->last_used != b->last_used)
|
||||
return a->last_used > b->last_used;
|
||||
else if (a->consumed != b->consumed)
|
||||
return a->consumed < b->consumed;
|
||||
else if (a->pending_consumed != b->pending_consumed)
|
||||
return a->pending_consumed > b->pending_consumed;
|
||||
else
|
||||
return a->id.id < b->id.id;
|
||||
});
|
||||
|
||||
size_t release_index = id_count;
|
||||
uint64_t activated_cost_this_iteration = 0;
|
||||
unsigned activation_count = 0;
|
||||
size_t activate_index = 0;
|
||||
|
||||
// Aim to activate resources as long as we're in budget.
|
||||
// Activate in order from highest priority to lowest.
|
||||
bool can_activate = true;
|
||||
while (can_activate &&
|
||||
total_consumed < transfer_budget &&
|
||||
activated_cost_this_iteration < transfer_budget_per_iteration &&
|
||||
activate_index != release_index)
|
||||
{
|
||||
auto *candidate = sorted_assets[activate_index];
|
||||
if (candidate->prio <= 0)
|
||||
break;
|
||||
|
||||
// This resource is already active.
|
||||
if (candidate->consumed != 0 || candidate->pending_consumed != 0)
|
||||
{
|
||||
activate_index++;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t estimate = iface->estimate_cost_asset(candidate->id, *candidate->handle);
|
||||
|
||||
can_activate = (total_consumed + estimate <= transfer_budget) || (candidate->prio >= persistent_prio());
|
||||
while (!can_activate && activate_index + 1 != release_index)
|
||||
{
|
||||
auto *release_candidate = sorted_assets[--release_index];
|
||||
if (release_candidate->consumed)
|
||||
{
|
||||
LOGI("Releasing ID %u due to page-in pressure.\n", release_candidate->id.id);
|
||||
iface->release_asset(release_candidate->id);
|
||||
total_consumed -= release_candidate->consumed;
|
||||
release_candidate->consumed = 0;
|
||||
}
|
||||
can_activate = total_consumed + estimate <= transfer_budget;
|
||||
}
|
||||
|
||||
if (can_activate)
|
||||
{
|
||||
// We're trivially in budget.
|
||||
iface->instantiate_asset(*this, task.get(), candidate->id, *candidate->handle);
|
||||
activation_count++;
|
||||
|
||||
candidate->pending_consumed = estimate;
|
||||
total_consumed += estimate;
|
||||
// Let this run over budget once.
|
||||
// Ensures we can make forward progress no matter what the limit is.
|
||||
activated_cost_this_iteration += estimate;
|
||||
activate_index++;
|
||||
}
|
||||
}
|
||||
|
||||
// If we're 75% of budget, start garbage collecting non-resident resources ahead of time.
|
||||
const uint64_t low_image_budget = (transfer_budget * 3) / 4;
|
||||
|
||||
const auto should_release = [&]() -> bool {
|
||||
if (release_index == activate_index)
|
||||
return false;
|
||||
if (sorted_assets[release_index - 1]->prio == persistent_prio())
|
||||
return false;
|
||||
|
||||
if (total_consumed > transfer_budget)
|
||||
return true;
|
||||
else if (total_consumed > low_image_budget && sorted_assets[release_index - 1]->prio == 0)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
// If we're over budget, deactivate resources.
|
||||
while (should_release())
|
||||
{
|
||||
auto *candidate = sorted_assets[--release_index];
|
||||
if (candidate->consumed)
|
||||
{
|
||||
LOGI("Releasing 0-prio ID %u due to page-in pressure.\n", candidate->id.id);
|
||||
iface->release_asset(candidate->id);
|
||||
total_consumed -= candidate->consumed;
|
||||
candidate->consumed = 0;
|
||||
candidate->last_used = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (activated_cost_this_iteration)
|
||||
{
|
||||
LOGI("Activated %u resources for %llu KiB.\n", activation_count,
|
||||
static_cast<unsigned long long>(activated_cost_this_iteration / 1024));
|
||||
}
|
||||
|
||||
iface->latch_handles();
|
||||
timestamp++;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "atomic_append_buffer.hpp"
|
||||
#include "global_managers.hpp"
|
||||
#include "filesystem.hpp"
|
||||
#include "object_pool.hpp"
|
||||
#include "intrusive_hash_map.hpp"
|
||||
#include "dynamic_array.hpp"
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
struct AssetID
|
||||
{
|
||||
uint32_t id = uint32_t(-1);
|
||||
enum { MaxIDs = 1u << 18 };
|
||||
AssetID() = default;
|
||||
explicit AssetID(uint32_t id_) : id{id_} {}
|
||||
explicit inline operator bool() const { return id != uint32_t(-1); }
|
||||
inline bool operator==(const AssetID &other) const { return id == other.id; }
|
||||
inline bool operator!=(const AssetID &other) const { return !(*this == other); }
|
||||
};
|
||||
|
||||
class AssetManager;
|
||||
|
||||
// If we have to fall back due to no image being present,
|
||||
// lets asset instantiator know what to substitute.
|
||||
enum class AssetClass
|
||||
{
|
||||
// Substitute with 0.
|
||||
ImageZeroable,
|
||||
// Substitute with missing color.
|
||||
ImageColor,
|
||||
// Substitute with RG8_UNORM 0.5
|
||||
ImageNormal,
|
||||
// Substitute with M = 0, R = 1.
|
||||
ImageMetallicRoughness,
|
||||
// Substitute with mid-gray (0.5, 0.5, 0.5, 1.0) UNORM8.
|
||||
// Somewhat compatible with everything.
|
||||
ImageGeneric,
|
||||
Mesh
|
||||
};
|
||||
|
||||
class ThreadGroup;
|
||||
struct TaskGroup;
|
||||
struct TaskSignal;
|
||||
|
||||
class AssetInstantiatorInterface
|
||||
{
|
||||
public:
|
||||
virtual ~AssetInstantiatorInterface() = default;
|
||||
|
||||
// This estimate should be an upper bound.
|
||||
virtual uint64_t estimate_cost_asset(AssetID id, File &mapping) = 0;
|
||||
|
||||
// When instantiation completes, manager.update_cost() must be called with the real cost.
|
||||
// The real cost may only be known after async parsing of the file.
|
||||
virtual void instantiate_asset(AssetManager &manager, TaskGroup *group, AssetID id, File &mapping) = 0;
|
||||
|
||||
// Will only be called after an upload completes through manager.update_cost().
|
||||
virtual void release_asset(AssetID id) = 0;
|
||||
|
||||
virtual void set_id_bounds(uint32_t bound) = 0;
|
||||
virtual void set_asset_class(AssetID id, AssetClass asset_class);
|
||||
|
||||
// Called in AssetManager::iterate().
|
||||
virtual void latch_handles() = 0;
|
||||
};
|
||||
|
||||
class AssetManager final : public AssetManagerInterface
|
||||
{
|
||||
public:
|
||||
// Persistent prio means the resource is treated as an internal LUT that must always be resident, no matter what.
|
||||
constexpr static int persistent_prio() { return 0x7fffffff; }
|
||||
|
||||
AssetManager();
|
||||
~AssetManager() override;
|
||||
|
||||
void set_asset_instantiator_interface(AssetInstantiatorInterface *iface);
|
||||
|
||||
// We might want to consider different budgets per asset class.
|
||||
void set_asset_budget(uint64_t cost);
|
||||
void set_asset_budget_per_iteration(uint64_t cost);
|
||||
|
||||
// FileHandle is intended to be used with FileSlice or similar here so that we don't need
|
||||
// a ton of open files at once.
|
||||
AssetID register_asset(FileHandle file, AssetClass asset_class, int prio = 1);
|
||||
AssetID register_asset(Filesystem &fs, const std::string &path, AssetClass asset_class, int prio = 1);
|
||||
|
||||
// Prio 0: Not resident, resource may not exist.
|
||||
bool set_asset_residency_priority(AssetID id, int prio);
|
||||
|
||||
// Intended to be called in Application::post_frame(). Not thread safe.
|
||||
// This function updates internal state.
|
||||
void iterate(ThreadGroup *group);
|
||||
bool iterate_blocking(ThreadGroup &group, AssetID id);
|
||||
|
||||
// Always thread safe, used by AssetInstantiatorInterfaces to update cost estimates.
|
||||
void update_cost(AssetID id, uint64_t cost);
|
||||
|
||||
// May be called concurrently, except when calling iterate().
|
||||
uint64_t get_current_total_consumed() const;
|
||||
|
||||
// May be called concurrently, except when calling iterate().
|
||||
// Intended to be called by asset instantiator interface or similar.
|
||||
// When a resource is actually accessed, this is called.
|
||||
void mark_used_asset(AssetID id);
|
||||
|
||||
// Should be called in applications's constructor to make sure we initialize
|
||||
// the mesh asset pool on device creation.
|
||||
// FIXME: Could be made more flexible if need be.
|
||||
void enable_mesh_assets();
|
||||
bool get_wants_mesh_assets() const;
|
||||
|
||||
private:
|
||||
struct AssetInfo : Util::IntrusiveHashMapEnabled<AssetInfo>
|
||||
{
|
||||
uint64_t pending_consumed = 0;
|
||||
uint64_t consumed = 0;
|
||||
uint64_t last_used = 0;
|
||||
FileHandle handle;
|
||||
AssetID id = {};
|
||||
AssetClass asset_class = AssetClass::ImageZeroable;
|
||||
int prio = 0;
|
||||
};
|
||||
|
||||
Util::DynamicArray<AssetInfo *> sorted_assets;
|
||||
Util::DynamicArray<AssetInfo *> asset_bank;
|
||||
std::mutex asset_bank_lock;
|
||||
Util::ObjectPool<AssetInfo> pool;
|
||||
Util::AtomicAppendBuffer<AssetID> lru_append;
|
||||
Util::IntrusiveHashMapHolder<AssetInfo> file_to_assets;
|
||||
|
||||
AssetInstantiatorInterface *iface = nullptr;
|
||||
uint32_t id_count = 0;
|
||||
uint64_t total_consumed = 0;
|
||||
uint64_t transfer_budget = 0;
|
||||
uint64_t transfer_budget_per_iteration = 0;
|
||||
uint64_t timestamp = 1;
|
||||
uint32_t blocking_signals = 0;
|
||||
|
||||
struct CostUpdate
|
||||
{
|
||||
AssetID id;
|
||||
uint64_t cost = 0;
|
||||
};
|
||||
std::mutex cost_update_lock;
|
||||
std::vector<CostUpdate> thread_cost_updates;
|
||||
std::vector<CostUpdate> cost_updates;
|
||||
|
||||
void adjust_update(const CostUpdate &update);
|
||||
std::unique_ptr<TaskSignal> signal;
|
||||
AssetID register_asset_nolock(FileHandle file, AssetClass asset_class, int prio);
|
||||
|
||||
void update_costs_locked_assets();
|
||||
void update_lru_locked_assets();
|
||||
|
||||
bool wants_mesh_assets = false;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,692 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#define NOMINMAX
|
||||
#include "filesystem.hpp"
|
||||
#include "path_utils.hpp"
|
||||
#include "logging.hpp"
|
||||
#include "os_filesystem.hpp"
|
||||
#include "string_helpers.hpp"
|
||||
#include "environment.hpp"
|
||||
#include <algorithm>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
std::vector<ListEntry> FilesystemBackend::walk(const std::string &path)
|
||||
{
|
||||
auto entries = list(path);
|
||||
std::vector<ListEntry> final_entries;
|
||||
for (auto &e : entries)
|
||||
{
|
||||
if (e.type == PathType::Directory)
|
||||
{
|
||||
auto subentries = walk(e.path);
|
||||
final_entries.push_back(std::move(e));
|
||||
for (auto &sub : subentries)
|
||||
final_entries.push_back(std::move(sub));
|
||||
}
|
||||
else if (e.type == PathType::File)
|
||||
final_entries.push_back(std::move(e));
|
||||
}
|
||||
return final_entries;
|
||||
}
|
||||
|
||||
bool FilesystemBackend::remove(const std::string &)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FilesystemBackend::move_replace(const std::string &, const std::string &)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FilesystemBackend::move_yield(const std::string &, const std::string &)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Filesystem::Filesystem()
|
||||
{
|
||||
register_protocol("file", std::unique_ptr<FilesystemBackend>(new OSFilesystem(".")));
|
||||
register_protocol("memory", std::unique_ptr<FilesystemBackend>(new ScratchFilesystem));
|
||||
|
||||
#ifdef GRANITE_DEFAULT_ASSET_DIRECTORY
|
||||
auto asset_dir = Util::get_environment_string("GRANITE_DEFAULT_ASSET_DIRECTORY", GRANITE_DEFAULT_ASSET_DIRECTORY);
|
||||
#else
|
||||
auto asset_dir = Util::get_environment_string("GRANITE_DEFAULT_ASSET_DIRECTORY", "");
|
||||
#endif
|
||||
if (!asset_dir.empty())
|
||||
register_protocol("builtin", std::unique_ptr<FilesystemBackend>(new OSFilesystem(asset_dir)));
|
||||
|
||||
#ifdef GRANITE_DEFAULT_BUILTIN_DIRECTORY
|
||||
auto builtin_dir = Util::get_environment_string("GRANITE_DEFAULT_BUILTIN_DIRECTORY", GRANITE_DEFAULT_BUILTIN_DIRECTORY);
|
||||
#else
|
||||
auto builtin_dir = Util::get_environment_string("GRANITE_DEFAULT_BUILTIN_DIRECTORY", "");
|
||||
#endif
|
||||
if (!builtin_dir.empty())
|
||||
register_protocol("builtin", std::unique_ptr<FilesystemBackend>(new OSFilesystem(builtin_dir)));
|
||||
|
||||
#ifdef GRANITE_DEFAULT_CACHE_DIRECTORY
|
||||
auto cache_dir = Util::get_environment_string("GRANITE_DEFAULT_CACHE_DIRECTORY", GRANITE_DEFAULT_CACHE_DIRECTORY);
|
||||
#else
|
||||
auto cache_dir = Util::get_environment_string("GRANITE_DEFAULT_CACHE_DIRECTORY", "");
|
||||
#endif
|
||||
if (!cache_dir.empty())
|
||||
register_protocol("cache", std::unique_ptr<FilesystemBackend>(new OSFilesystem(cache_dir)));
|
||||
}
|
||||
|
||||
void Filesystem::setup_default_filesystem(Filesystem *filesystem, const char *default_asset_directory)
|
||||
{
|
||||
auto self_dir = Path::basedir(Path::get_executable_path());
|
||||
auto assets_dir = Path::join(self_dir, "assets");
|
||||
auto builtin_dir = Path::join(self_dir, "builtin/assets");
|
||||
|
||||
if (default_asset_directory)
|
||||
{
|
||||
#ifdef GRANITE_SHIPPING
|
||||
LOGW("Default asset directory %s was provided, but this is only intended for non-shipping configs.\n",
|
||||
default_asset_directory);
|
||||
#else
|
||||
filesystem->register_protocol("assets",
|
||||
std::unique_ptr<FilesystemBackend>(new OSFilesystem(default_asset_directory)));
|
||||
#endif
|
||||
}
|
||||
|
||||
FileStat s;
|
||||
if (filesystem->stat(assets_dir, s) && s.type == PathType::Directory)
|
||||
{
|
||||
filesystem->register_protocol("assets", std::make_unique<OSFilesystem>(assets_dir));
|
||||
LOGI("Redirecting filesystem \"assets\" to %s.\n", assets_dir.c_str());
|
||||
|
||||
auto cache_dir = Path::join(self_dir, "cache");
|
||||
filesystem->register_protocol("cache", std::make_unique<OSFilesystem>(cache_dir));
|
||||
LOGI("Redirecting filesystem \"cache\" to %s.\n", cache_dir.c_str());
|
||||
}
|
||||
|
||||
if (filesystem->stat(builtin_dir, s) && s.type == PathType::Directory)
|
||||
{
|
||||
filesystem->register_protocol("builtin", std::make_unique<OSFilesystem>(builtin_dir));
|
||||
LOGI("Redirecting filesystem \"builtin\" to %s.\n", builtin_dir.c_str());
|
||||
}
|
||||
|
||||
// These filesystems are core functionality.
|
||||
if (!filesystem->get_backend("builtin"))
|
||||
throw std::runtime_error("builtin filesystem was not initialized.");
|
||||
if (!filesystem->get_backend("cache"))
|
||||
throw std::runtime_error("cache filesystem was not initialized.");
|
||||
}
|
||||
|
||||
void Filesystem::register_protocol(const std::string &proto, std::unique_ptr<FilesystemBackend> fs)
|
||||
{
|
||||
if (fs)
|
||||
{
|
||||
fs->set_protocol(proto);
|
||||
protocols[proto] = std::move(fs);
|
||||
}
|
||||
else
|
||||
protocols.erase(proto);
|
||||
}
|
||||
|
||||
FilesystemBackend *Filesystem::get_backend(const std::string &proto)
|
||||
{
|
||||
auto itr = protocols.find(proto);
|
||||
if (proto.empty())
|
||||
itr = protocols.find("file");
|
||||
|
||||
if (itr != end(protocols))
|
||||
return itr->second.get();
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<ListEntry> Filesystem::walk(const std::string &path)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return {};
|
||||
|
||||
return backend->walk(paths.second);
|
||||
}
|
||||
|
||||
std::vector<ListEntry> Filesystem::list(const std::string &path)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return {};
|
||||
|
||||
return backend->list(paths.second);
|
||||
}
|
||||
|
||||
bool Filesystem::remove(const std::string &path)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return false;
|
||||
|
||||
return backend->remove(paths.second);
|
||||
}
|
||||
|
||||
bool Filesystem::move_yield(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto paths_dst = Path::protocol_split(dst);
|
||||
auto paths_src = Path::protocol_split(src);
|
||||
auto *backend_dst = get_backend(paths_dst.first);
|
||||
auto *backend_src = get_backend(paths_src.first);
|
||||
if (!backend_dst || !backend_src || backend_dst != backend_src)
|
||||
return false;
|
||||
|
||||
return backend_dst->move_yield(paths_dst.second, paths_src.second);
|
||||
}
|
||||
|
||||
bool Filesystem::move_replace(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto paths_dst = Path::protocol_split(dst);
|
||||
auto paths_src = Path::protocol_split(src);
|
||||
auto *backend_dst = get_backend(paths_dst.first);
|
||||
auto *backend_src = get_backend(paths_src.first);
|
||||
if (!backend_dst || !backend_src || backend_dst != backend_src)
|
||||
return false;
|
||||
|
||||
return backend_dst->move_replace(paths_dst.second, paths_src.second);
|
||||
}
|
||||
|
||||
FileMappingHandle Filesystem::open_readonly_mapping(const std::string &path)
|
||||
{
|
||||
auto file = open(path, FileMode::ReadOnly);
|
||||
if (!file)
|
||||
return {};
|
||||
return file->map();
|
||||
}
|
||||
|
||||
FileMappingHandle Filesystem::open_writeonly_mapping(const std::string &path, size_t size)
|
||||
{
|
||||
auto file = open(path, FileMode::WriteOnly);
|
||||
if (!file)
|
||||
return {};
|
||||
return file->map_write(size);
|
||||
}
|
||||
|
||||
FileMappingHandle Filesystem::open_transactional_mapping(const std::string &path, size_t size)
|
||||
{
|
||||
auto file = open(path, FileMode::WriteOnlyTransactional);
|
||||
if (!file)
|
||||
return {};
|
||||
return file->map_write(size);
|
||||
}
|
||||
|
||||
bool Filesystem::read_file_to_string(const std::string &path, std::string &str)
|
||||
{
|
||||
auto mapping = open_readonly_mapping(path);
|
||||
if (!mapping)
|
||||
return false;
|
||||
|
||||
auto size = mapping->get_size();
|
||||
str = std::string(mapping->data<char>(), mapping->data<char>() + size);
|
||||
|
||||
// Remove DOS EOL.
|
||||
str.erase(remove_if(begin(str), end(str), [](char c) { return c == '\r'; }), end(str));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Filesystem::write_buffer_to_file(const std::string &path, const void *data, size_t size)
|
||||
{
|
||||
auto file = open_transactional_mapping(path, size);
|
||||
if (!file)
|
||||
return false;
|
||||
memcpy(file->mutable_data(), data, size);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Filesystem::write_string_to_file(const std::string &path, const std::string &str)
|
||||
{
|
||||
return write_buffer_to_file(path, str.data(), str.size());
|
||||
}
|
||||
|
||||
FileHandle Filesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return {};
|
||||
|
||||
auto file = backend->open(paths.second, mode);
|
||||
return file;
|
||||
}
|
||||
|
||||
std::string Filesystem::get_filesystem_path(const std::string &path)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return "";
|
||||
|
||||
return backend->get_filesystem_path(paths.second);
|
||||
}
|
||||
|
||||
bool Filesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *backend = get_backend(paths.first);
|
||||
if (!backend)
|
||||
return false;
|
||||
|
||||
return backend->stat(paths.second, stat);
|
||||
}
|
||||
|
||||
void Filesystem::poll_notifications()
|
||||
{
|
||||
for (auto &proto : protocols)
|
||||
proto.second->poll_notifications();
|
||||
}
|
||||
|
||||
bool Filesystem::load_text_file(const std::string &path, std::string &str)
|
||||
{
|
||||
return read_file_to_string(path, str);
|
||||
}
|
||||
|
||||
int ScratchFilesystem::get_notification_fd() const
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
FileNotifyHandle ScratchFilesystem::install_notification(const std::string &,
|
||||
std::function<void(const FileNotifyInfo &)>)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
void ScratchFilesystem::poll_notifications()
|
||||
{
|
||||
}
|
||||
|
||||
void ScratchFilesystem::uninstall_notification(FileNotifyHandle)
|
||||
{
|
||||
}
|
||||
|
||||
bool ScratchFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto itr = scratch_files.find(path);
|
||||
if (itr == end(scratch_files))
|
||||
return false;
|
||||
|
||||
stat.size = itr->second->data.size();
|
||||
stat.type = PathType::File;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<ListEntry> ScratchFilesystem::list(const std::string &)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
struct ScratchFilesystemFile final : File
|
||||
{
|
||||
explicit ScratchFilesystemFile(std::vector<uint8_t> &data_)
|
||||
: data(data_)
|
||||
{
|
||||
}
|
||||
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t range) override
|
||||
{
|
||||
if (offset + range > data.size())
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
FileHandle{}, offset,
|
||||
data.data() + offset, range,
|
||||
0, range);
|
||||
}
|
||||
|
||||
FileMappingHandle map_write(size_t size) override
|
||||
{
|
||||
data.resize(size);
|
||||
return map_subset(0, size);
|
||||
}
|
||||
|
||||
void unmap(void *, size_t) override
|
||||
{
|
||||
}
|
||||
|
||||
uint64_t get_size() override
|
||||
{
|
||||
return data.size();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> &data;
|
||||
};
|
||||
|
||||
FileHandle ScratchFilesystem::open(const std::string &path, FileMode)
|
||||
{
|
||||
auto itr = scratch_files.find(path);
|
||||
if (itr == end(scratch_files))
|
||||
{
|
||||
auto &file = scratch_files[path];
|
||||
file = std::make_unique<ScratchFile>();
|
||||
return Util::make_handle<ScratchFilesystemFile>(file->data);
|
||||
}
|
||||
else
|
||||
{
|
||||
return Util::make_handle<ScratchFilesystemFile>(itr->second->data);
|
||||
}
|
||||
}
|
||||
|
||||
BlobFilesystem::BlobFilesystem(FileHandle file_)
|
||||
: file(std::move(file_))
|
||||
{
|
||||
if (!file)
|
||||
return;
|
||||
|
||||
root = std::make_unique<Directory>();
|
||||
parse();
|
||||
}
|
||||
|
||||
uint8_t BlobFilesystem::read_u8(const uint8_t *&buf, size_t &size)
|
||||
{
|
||||
if (size < 1)
|
||||
throw std::range_error("Blob EOF.");
|
||||
|
||||
uint8_t ret = *buf++;
|
||||
size--;
|
||||
return ret;
|
||||
}
|
||||
|
||||
uint64_t BlobFilesystem::read_u64(const uint8_t *&buf, size_t &size)
|
||||
{
|
||||
if (size < 8)
|
||||
throw std::range_error("Blob EOF.");
|
||||
|
||||
uint64_t ret = 0;
|
||||
for (unsigned i = 0; i < 8; i++)
|
||||
ret |= uint64_t(buf[i]) << (8 * i);
|
||||
size -= 8;
|
||||
buf += 8;
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string BlobFilesystem::read_string(const uint8_t *&buf, size_t &size, size_t len)
|
||||
{
|
||||
if (size < len)
|
||||
throw std::range_error("Blob EOF.");
|
||||
|
||||
std::string ret;
|
||||
ret.insert(ret.end(), reinterpret_cast<const char *>(buf), reinterpret_cast<const char *>(buf) + len);
|
||||
size -= len;
|
||||
buf += len;
|
||||
return ret;
|
||||
}
|
||||
|
||||
void BlobFilesystem::add_entry(const std::string &path, size_t offset, size_t size)
|
||||
{
|
||||
auto paths = Path::split(path);
|
||||
auto *dir = find_directory(paths.first);
|
||||
if (!dir)
|
||||
dir = make_directory(paths.first);
|
||||
dir->files.push_back({ Path::basename(path), offset, size });
|
||||
}
|
||||
|
||||
void BlobFilesystem::parse()
|
||||
{
|
||||
size_t mapped_size = file->get_size();
|
||||
if (mapped_size < 16)
|
||||
throw std::runtime_error("Blob archive too small.");
|
||||
|
||||
auto mapped_handle = file->map();
|
||||
if (!mapped_handle)
|
||||
throw std::runtime_error("Failed to map blob archive.");
|
||||
|
||||
auto *base_mapped = mapped_handle->data<uint8_t>();
|
||||
auto *mapped = base_mapped;
|
||||
|
||||
if (memcmp(mapped, "BLOBBY01", 8) != 0)
|
||||
throw std::runtime_error("Invalid magic.");
|
||||
mapped += 8;
|
||||
mapped_size -= 8;
|
||||
|
||||
uint64_t required_size = 0;
|
||||
|
||||
while (mapped_size >= 4 && memcmp(mapped, "ENTR", 4) == 0)
|
||||
{
|
||||
mapped += 4;
|
||||
mapped_size -= 4;
|
||||
|
||||
uint8_t len = read_u8(mapped, mapped_size);
|
||||
std::string path = Path::canonicalize_path(read_string(mapped, mapped_size, len));
|
||||
uint64_t blob_offset = read_u64(mapped, mapped_size);
|
||||
uint64_t blob_size = read_u64(mapped, mapped_size);
|
||||
required_size = std::max(required_size, blob_offset + blob_size);
|
||||
|
||||
if (blob_offset + blob_size < blob_offset)
|
||||
throw std::range_error("Overflow for blob offset + size.");
|
||||
|
||||
if (blob_offset > SIZE_MAX || blob_size > SIZE_MAX)
|
||||
throw std::range_error("Blob offset out of range.");
|
||||
|
||||
add_entry(path, blob_offset, blob_size);
|
||||
}
|
||||
|
||||
if (mapped_size >= 4 && memcmp(mapped, "DATA", 4) == 0)
|
||||
{
|
||||
blob_base_offset = size_t((mapped + 4) - base_mapped);
|
||||
mapped_size -= 4;
|
||||
if (mapped_size < required_size)
|
||||
throw std::range_error("Blob is not large enough for all files.");
|
||||
}
|
||||
}
|
||||
|
||||
BlobFilesystem::Directory *BlobFilesystem::make_directory(const std::string &path)
|
||||
{
|
||||
auto split = Util::split_no_empty(path, "/");
|
||||
auto *dir = root.get();
|
||||
|
||||
for (const auto &subpath : split)
|
||||
{
|
||||
auto dir_itr = std::find_if(dir->dirs.begin(), dir->dirs.end(), [&](const std::unique_ptr<Directory> &dir_) {
|
||||
return subpath == dir_->path;
|
||||
});
|
||||
|
||||
if (dir_itr != dir->dirs.end())
|
||||
dir = dir_itr->get();
|
||||
else
|
||||
{
|
||||
dir->dirs.emplace_back(new Directory);
|
||||
dir->dirs.back()->path = subpath;
|
||||
dir = dir->dirs.back().get();
|
||||
}
|
||||
}
|
||||
|
||||
return dir;
|
||||
}
|
||||
|
||||
BlobFilesystem::Directory *BlobFilesystem::find_directory(const std::string &path)
|
||||
{
|
||||
auto split = Util::split_no_empty(path, "/");
|
||||
auto *dir = root.get();
|
||||
|
||||
for (const auto &subpath : split)
|
||||
{
|
||||
auto dir_itr = std::find_if(dir->dirs.begin(), dir->dirs.end(), [&](const std::unique_ptr<Directory> &dir_) {
|
||||
return subpath == dir_->path;
|
||||
});
|
||||
|
||||
if (dir_itr != dir->dirs.end())
|
||||
dir = dir_itr->get();
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
return dir;
|
||||
}
|
||||
|
||||
BlobFilesystem::BlobFile *BlobFilesystem::find_file(const std::string &path)
|
||||
{
|
||||
auto paths = Path::split(path);
|
||||
auto *dir = find_directory(paths.first);
|
||||
if (!dir)
|
||||
return nullptr;
|
||||
|
||||
auto file_itr = std::find_if(dir->files.begin(), dir->files.end(), [&](const BlobFile &zip_file) {
|
||||
return paths.second == zip_file.path;
|
||||
});
|
||||
|
||||
if (file_itr != dir->files.end())
|
||||
return &*file_itr;
|
||||
else
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<ListEntry> BlobFilesystem::list(const std::string &path)
|
||||
{
|
||||
auto canon_path = Path::canonicalize_path(path);
|
||||
|
||||
std::vector<ListEntry> entries;
|
||||
if (const auto *zip_dir = find_directory(canon_path))
|
||||
{
|
||||
entries.reserve(zip_dir->dirs.size() + zip_dir->files.size());
|
||||
for (auto &dir : zip_dir->dirs)
|
||||
entries.push_back({ Path::join(path, dir->path), PathType::Directory });
|
||||
for (auto &f : zip_dir->files)
|
||||
entries.push_back({ Path::join(path, f.path), PathType::File });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
bool BlobFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto p = Path::canonicalize_path(path);
|
||||
|
||||
if (const auto *zip_file = find_file(p))
|
||||
{
|
||||
stat.size = zip_file->size;
|
||||
stat.type = PathType::File;
|
||||
stat.last_modified = 0;
|
||||
return true;
|
||||
}
|
||||
else if (find_directory(p))
|
||||
{
|
||||
stat.size = 0;
|
||||
stat.last_modified = 0;
|
||||
stat.type = PathType::Directory;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
FileHandle BlobFilesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
if (mode != FileMode::ReadOnly)
|
||||
return {};
|
||||
|
||||
auto p = Path::canonicalize_path(path);
|
||||
auto *blob_file = find_file(p);
|
||||
if (!blob_file)
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileSlice>(file, blob_base_offset + blob_file->offset, blob_file->size);
|
||||
}
|
||||
|
||||
FileNotifyHandle BlobFilesystem::install_notification(const std::string &, std::function<void (const FileNotifyInfo &)>)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
void BlobFilesystem::uninstall_notification(FileNotifyHandle)
|
||||
{
|
||||
}
|
||||
|
||||
void BlobFilesystem::poll_notifications()
|
||||
{
|
||||
}
|
||||
|
||||
int BlobFilesystem::get_notification_fd() const
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
FileMapping::FileMapping(FileHandle handle_, uint64_t file_offset_,
|
||||
void *mapped_, size_t mapped_size_,
|
||||
size_t map_offset_, size_t accessible_size_)
|
||||
: handle(std::move(handle_))
|
||||
, file_offset(file_offset_)
|
||||
, mapped(mapped_)
|
||||
, mapped_size(mapped_size_)
|
||||
, map_offset(map_offset_)
|
||||
, accessible_size(accessible_size_)
|
||||
{
|
||||
}
|
||||
|
||||
FileMapping::~FileMapping()
|
||||
{
|
||||
if (handle)
|
||||
handle->unmap(mapped, mapped_size);
|
||||
}
|
||||
|
||||
uint64_t FileMapping::get_file_offset() const
|
||||
{
|
||||
return file_offset;
|
||||
}
|
||||
|
||||
uint64_t FileMapping::get_size() const
|
||||
{
|
||||
return accessible_size;
|
||||
}
|
||||
|
||||
Util::IntrusivePtr<FileMapping> File::map()
|
||||
{
|
||||
return map_subset(0, get_size());
|
||||
}
|
||||
|
||||
FileSlice::FileSlice(FileHandle handle_, uint64_t offset_, uint64_t range_)
|
||||
: handle(std::move(handle_)), offset(offset_), range(range_)
|
||||
{
|
||||
}
|
||||
|
||||
FileMappingHandle FileSlice::map_subset(uint64_t offset_, size_t range_)
|
||||
{
|
||||
if (offset_ + range_ > range)
|
||||
return {};
|
||||
return handle->map_subset(offset + offset_, range_);
|
||||
}
|
||||
|
||||
FileMappingHandle FileSlice::map_write(size_t)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
uint64_t FileSlice::get_size()
|
||||
{
|
||||
return range;
|
||||
}
|
||||
|
||||
void FileSlice::unmap(void *mapped, size_t mapped_size)
|
||||
{
|
||||
handle->unmap(mapped, mapped_size);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <functional>
|
||||
#include <stdio.h>
|
||||
#include "global_managers.hpp"
|
||||
#include "intrusive.hpp"
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
class FileMapping;
|
||||
|
||||
class File : public Util::ThreadSafeIntrusivePtrEnabled<File>
|
||||
{
|
||||
public:
|
||||
virtual ~File() = default;
|
||||
virtual Util::IntrusivePtr<FileMapping> map_subset(uint64_t offset, size_t range) = 0;
|
||||
virtual Util::IntrusivePtr<FileMapping> map_write(size_t size) = 0;
|
||||
virtual uint64_t get_size() = 0;
|
||||
|
||||
// Only called by FileMapping.
|
||||
virtual void unmap(void *mapped, size_t range) = 0;
|
||||
|
||||
Util::IntrusivePtr<FileMapping> map();
|
||||
};
|
||||
using FileHandle = Util::IntrusivePtr<File>;
|
||||
|
||||
class FileMapping : public Util::ThreadSafeIntrusivePtrEnabled<FileMapping>
|
||||
{
|
||||
public:
|
||||
template <typename T = void>
|
||||
inline const T *data() const
|
||||
{
|
||||
void *ptr = static_cast<uint8_t *>(mapped) + map_offset;
|
||||
return static_cast<const T *>(ptr);
|
||||
}
|
||||
|
||||
template <typename T = void>
|
||||
inline T *mutable_data()
|
||||
{
|
||||
void *ptr = static_cast<uint8_t *>(mapped) + map_offset;
|
||||
return static_cast<T *>(ptr);
|
||||
}
|
||||
|
||||
uint64_t get_file_offset() const;
|
||||
uint64_t get_size() const;
|
||||
|
||||
~FileMapping();
|
||||
FileMapping(FileHandle handle,
|
||||
uint64_t file_offset,
|
||||
void *mapped, size_t mapped_size,
|
||||
size_t map_offset, size_t accessible_size);
|
||||
|
||||
private:
|
||||
FileHandle handle;
|
||||
uint64_t file_offset;
|
||||
void *mapped;
|
||||
size_t mapped_size;
|
||||
// For non-page aligned maps.
|
||||
size_t map_offset;
|
||||
size_t accessible_size;
|
||||
};
|
||||
using FileMappingHandle = Util::IntrusivePtr<FileMapping>;
|
||||
|
||||
enum class PathType
|
||||
{
|
||||
File,
|
||||
Directory,
|
||||
Special
|
||||
};
|
||||
|
||||
struct ListEntry
|
||||
{
|
||||
std::string path;
|
||||
PathType type;
|
||||
};
|
||||
|
||||
struct FileStat
|
||||
{
|
||||
uint64_t size;
|
||||
PathType type;
|
||||
uint64_t last_modified;
|
||||
};
|
||||
|
||||
using FileNotifyHandle = int;
|
||||
|
||||
enum class FileNotifyType
|
||||
{
|
||||
FileChanged,
|
||||
FileDeleted,
|
||||
FileCreated,
|
||||
};
|
||||
|
||||
struct FileNotifyInfo
|
||||
{
|
||||
std::string path;
|
||||
FileNotifyType type;
|
||||
FileNotifyHandle handle;
|
||||
};
|
||||
|
||||
enum class FileMode
|
||||
{
|
||||
ReadOnly,
|
||||
WriteOnly,
|
||||
ReadWrite,
|
||||
WriteOnlyTransactional
|
||||
};
|
||||
|
||||
class FilesystemBackend
|
||||
{
|
||||
public:
|
||||
virtual ~FilesystemBackend() = default;
|
||||
std::vector<ListEntry> walk(const std::string &path);
|
||||
virtual std::vector<ListEntry> list(const std::string &path) = 0;
|
||||
virtual FileHandle open(const std::string &path, FileMode mode = FileMode::ReadOnly) = 0;
|
||||
virtual bool stat(const std::string &path, FileStat &stat) = 0;
|
||||
|
||||
virtual FileNotifyHandle
|
||||
install_notification(const std::string &path, std::function<void(const FileNotifyInfo &)> func) = 0;
|
||||
|
||||
virtual void uninstall_notification(FileNotifyHandle handle) = 0;
|
||||
virtual void poll_notifications() = 0;
|
||||
virtual int get_notification_fd() const = 0;
|
||||
|
||||
virtual bool remove(const std::string &path);
|
||||
virtual bool move_replace(const std::string &dst, const std::string &src);
|
||||
virtual bool move_yield(const std::string &dst, const std::string &src);
|
||||
|
||||
inline virtual std::string get_filesystem_path(const std::string &)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
void set_protocol(const std::string &proto)
|
||||
{
|
||||
protocol = proto;
|
||||
}
|
||||
|
||||
protected:
|
||||
std::string protocol;
|
||||
};
|
||||
|
||||
class Filesystem final : public FilesystemInterface
|
||||
{
|
||||
public:
|
||||
Filesystem();
|
||||
|
||||
void register_protocol(const std::string &proto, std::unique_ptr<FilesystemBackend> fs);
|
||||
FilesystemBackend *get_backend(const std::string &proto);
|
||||
std::vector<ListEntry> walk(const std::string &path);
|
||||
std::vector<ListEntry> list(const std::string &path);
|
||||
FileHandle open(const std::string &path, FileMode mode = FileMode::ReadOnly);
|
||||
std::string get_filesystem_path(const std::string &path);
|
||||
|
||||
bool read_file_to_string(const std::string &path, std::string &str);
|
||||
bool write_string_to_file(const std::string &path, const std::string &str);
|
||||
bool write_buffer_to_file(const std::string &path, const void *data, size_t size);
|
||||
FileMappingHandle open_readonly_mapping(const std::string &path);
|
||||
FileMappingHandle open_writeonly_mapping(const std::string &path, size_t size);
|
||||
FileMappingHandle open_transactional_mapping(const std::string &path, size_t size);
|
||||
|
||||
bool remove(const std::string &path);
|
||||
bool move_replace(const std::string &dst, const std::string &src);
|
||||
bool move_yield(const std::string &dst, const std::string &src);
|
||||
|
||||
bool stat(const std::string &path, FileStat &stat);
|
||||
|
||||
void poll_notifications();
|
||||
|
||||
const std::unordered_map<std::string, std::unique_ptr<FilesystemBackend>> &get_protocols() const
|
||||
{
|
||||
return protocols;
|
||||
}
|
||||
|
||||
static void setup_default_filesystem(Filesystem *fs, const char *default_asset_directory);
|
||||
|
||||
private:
|
||||
std::unordered_map<std::string, std::unique_ptr<FilesystemBackend>> protocols;
|
||||
|
||||
bool load_text_file(const std::string &path, std::string &str) override;
|
||||
};
|
||||
|
||||
class ScratchFilesystem final : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
|
||||
FileHandle open(const std::string &path, FileMode mode = FileMode::ReadOnly) override;
|
||||
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void(const FileNotifyInfo &)> func) override;
|
||||
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
|
||||
void poll_notifications() override;
|
||||
|
||||
int get_notification_fd() const override;
|
||||
|
||||
private:
|
||||
struct ScratchFile
|
||||
{
|
||||
std::vector<uint8_t> data;
|
||||
};
|
||||
std::unordered_map<std::string, std::unique_ptr<ScratchFile>> scratch_files;
|
||||
};
|
||||
|
||||
class ConstantMemoryFile final : public Granite::File
|
||||
{
|
||||
public:
|
||||
ConstantMemoryFile(const void *mapped_, size_t size_)
|
||||
: mapped(static_cast<const uint8_t *>(mapped_)), size(size_)
|
||||
{
|
||||
}
|
||||
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t range) override
|
||||
{
|
||||
if (offset + range > size)
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
FileHandle{}, offset,
|
||||
const_cast<uint8_t *>(mapped) + offset, range,
|
||||
0, range);
|
||||
}
|
||||
|
||||
FileMappingHandle map_write(size_t) override
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
void unmap(void *, size_t) override
|
||||
{
|
||||
}
|
||||
|
||||
uint64_t get_size() override
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
private:
|
||||
const uint8_t *mapped;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
class FileSlice final : public File
|
||||
{
|
||||
public:
|
||||
FileSlice(FileHandle handle, uint64_t offset, uint64_t range);
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t range) override;
|
||||
FileMappingHandle map_write(size_t) override;
|
||||
void unmap(void *, size_t) override;
|
||||
uint64_t get_size() override;
|
||||
|
||||
private:
|
||||
FileHandle handle;
|
||||
uint64_t offset;
|
||||
uint64_t range;
|
||||
};
|
||||
|
||||
class BlobFilesystem final : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
BlobFilesystem(FileHandle file);
|
||||
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
|
||||
FileHandle open(const std::string &path, FileMode mode) override;
|
||||
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void(const FileNotifyInfo &)> func) override;
|
||||
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
|
||||
void poll_notifications() override;
|
||||
|
||||
int get_notification_fd() const override;
|
||||
|
||||
private:
|
||||
FileHandle file;
|
||||
size_t blob_base_offset = 0;
|
||||
|
||||
struct BlobFile
|
||||
{
|
||||
std::string path;
|
||||
size_t offset;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
struct Directory
|
||||
{
|
||||
std::string path;
|
||||
std::vector<std::unique_ptr<Directory>> dirs;
|
||||
std::vector<BlobFile> files;
|
||||
};
|
||||
std::unique_ptr<Directory> root;
|
||||
BlobFile *find_file(const std::string &path);
|
||||
Directory *find_directory(const std::string &path);
|
||||
Directory *make_directory(const std::string &path);
|
||||
void parse();
|
||||
|
||||
static uint8_t read_u8(const uint8_t *&buf, size_t &size);
|
||||
static uint64_t read_u64(const uint8_t *&buf, size_t &size);
|
||||
static std::string read_string(const uint8_t *&buf, size_t &size, size_t len);
|
||||
void add_entry(const std::string &path, size_t offset, size_t size);
|
||||
};
|
||||
|
||||
}
|
||||
+541
@@ -0,0 +1,541 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "os_filesystem.hpp"
|
||||
#include "path_utils.hpp"
|
||||
#include "logging.hpp"
|
||||
#include <algorithm>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <errno.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <string.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <atomic>
|
||||
#ifdef __linux__
|
||||
#include <sys/inotify.h>
|
||||
#endif
|
||||
|
||||
#if defined(__FreeBSD__) || defined(__APPLE__)
|
||||
#define FSTAT64 fstat
|
||||
#define FTRUNCATE64 ftruncate
|
||||
#define MMAP64 mmap
|
||||
#define STAT64 stat
|
||||
#define off64_t off_t
|
||||
#else
|
||||
#define FSTAT64 fstat64
|
||||
#define FTRUNCATE64 ftruncate64
|
||||
#define MMAP64 mmap64
|
||||
#define STAT64 stat64
|
||||
#endif
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
static bool ensure_directory_inner(const std::string &path)
|
||||
{
|
||||
if (Path::is_root_path(path))
|
||||
return false;
|
||||
|
||||
struct STAT64 s = {};
|
||||
if (::STAT64(path.c_str(), &s) >= 0 && S_ISDIR(s.st_mode))
|
||||
return true;
|
||||
|
||||
auto basedir = Path::basedir(path);
|
||||
if (!ensure_directory_inner(basedir))
|
||||
return false;
|
||||
|
||||
return (mkdir(path.c_str(), 0750) >= 0) || (errno == EEXIST);
|
||||
}
|
||||
|
||||
static bool ensure_directory(const std::string &path)
|
||||
{
|
||||
auto basedir = Path::basedir(path);
|
||||
return ensure_directory_inner(basedir);
|
||||
}
|
||||
|
||||
FileHandle MMapFile::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
auto file = Util::make_handle<MMapFile>();
|
||||
if (!file->init(path, mode))
|
||||
file.reset();
|
||||
return file;
|
||||
}
|
||||
|
||||
static std::atomic_uint32_t global_transaction_counter;
|
||||
|
||||
bool MMapFile::init(const std::string &path, FileMode mode)
|
||||
{
|
||||
int modeflags = 0;
|
||||
switch (mode)
|
||||
{
|
||||
case FileMode::ReadOnly:
|
||||
modeflags = O_RDONLY;
|
||||
break;
|
||||
|
||||
case FileMode::WriteOnly:
|
||||
case FileMode::WriteOnlyTransactional:
|
||||
if (!ensure_directory(path))
|
||||
{
|
||||
LOGE("MMapFile failed to create directory.\n");
|
||||
return false;
|
||||
}
|
||||
modeflags = O_RDWR | O_CREAT | O_TRUNC; // Need read access for mmap.
|
||||
break;
|
||||
|
||||
case FileMode::ReadWrite:
|
||||
if (!ensure_directory(path))
|
||||
{
|
||||
LOGE("MMapFile failed to create directory.\n");
|
||||
return false;
|
||||
}
|
||||
modeflags = O_RDWR | O_CREAT;
|
||||
break;
|
||||
}
|
||||
|
||||
const char *open_path = path.c_str();
|
||||
|
||||
if (mode == FileMode::WriteOnlyTransactional)
|
||||
{
|
||||
// Use atomic file rename to ensure that a file is written atomically.
|
||||
rename_to_on_close = path;
|
||||
rename_from_on_close =
|
||||
path + ".tmp." +
|
||||
std::to_string(getpid()) + "." +
|
||||
std::to_string(global_transaction_counter.fetch_add(1, std::memory_order_relaxed));
|
||||
open_path = rename_from_on_close.c_str();
|
||||
}
|
||||
|
||||
fd = ::open(open_path, modeflags, 0640);
|
||||
if (fd < 0)
|
||||
{
|
||||
rename_to_on_close.clear();
|
||||
rename_from_on_close.clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!query_stat())
|
||||
{
|
||||
close(fd);
|
||||
rename_to_on_close.clear();
|
||||
rename_from_on_close.clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
FileMappingHandle MMapFile::map_write(size_t map_size)
|
||||
{
|
||||
if (has_write_map)
|
||||
return {};
|
||||
|
||||
if (FTRUNCATE64(fd, off64_t(map_size)) < 0)
|
||||
{
|
||||
LOGE("Failed to truncate.\n");
|
||||
report_error();
|
||||
return {};
|
||||
}
|
||||
|
||||
size = map_size;
|
||||
|
||||
void *mapped = MMAP64(nullptr, map_size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
|
||||
if (mapped == MAP_FAILED)
|
||||
{
|
||||
report_error();
|
||||
return {};
|
||||
}
|
||||
|
||||
has_write_map = true;
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
reference_from_this(),
|
||||
0,
|
||||
mapped, map_size,
|
||||
0, map_size);
|
||||
}
|
||||
|
||||
void MMapFile::report_error()
|
||||
{
|
||||
#ifdef __linux__
|
||||
int err = errno;
|
||||
char fdpath[PATH_MAX];
|
||||
char path[PATH_MAX];
|
||||
snprintf(fdpath, sizeof(fdpath), "/proc/%u/fd/%d", getpid(), fd);
|
||||
int ret = readlink(fdpath, path, sizeof(path) - 1);
|
||||
if (ret > 0)
|
||||
{
|
||||
path[ret] = '\0';
|
||||
LOGE("mmap failed for \"%s\" (%s).\n", path, strerror(err));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
FileMappingHandle MMapFile::map_subset(uint64_t offset, size_t range)
|
||||
{
|
||||
uint64_t page_size = sysconf(_SC_PAGESIZE);
|
||||
uint64_t begin_map = offset & ~(page_size - 1);
|
||||
uint64_t end_map = offset + range;
|
||||
size_t mapped_size = end_map - begin_map;
|
||||
|
||||
// length need not be aligned.
|
||||
|
||||
void *mapped = MMAP64(nullptr, mapped_size, PROT_READ, MAP_PRIVATE, fd, off64_t(begin_map));
|
||||
if (mapped == MAP_FAILED)
|
||||
{
|
||||
report_error();
|
||||
return {};
|
||||
}
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
reference_from_this(),
|
||||
offset,
|
||||
mapped, mapped_size,
|
||||
offset - begin_map, range);
|
||||
}
|
||||
|
||||
uint64_t MMapFile::get_size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
bool MMapFile::query_stat()
|
||||
{
|
||||
struct STAT64 s = {};
|
||||
if (FSTAT64(fd, &s) < 0)
|
||||
return false;
|
||||
|
||||
if (uint64_t(s.st_size) > SIZE_MAX)
|
||||
return false;
|
||||
size = static_cast<size_t>(s.st_size);
|
||||
return true;
|
||||
}
|
||||
|
||||
void MMapFile::unmap(void *mapped, size_t mapped_size)
|
||||
{
|
||||
munmap(mapped, mapped_size);
|
||||
}
|
||||
|
||||
MMapFile::~MMapFile()
|
||||
{
|
||||
if (fd >= 0)
|
||||
close(fd);
|
||||
|
||||
if (!rename_from_on_close.empty() && !rename_to_on_close.empty())
|
||||
{
|
||||
int ret = rename(rename_from_on_close.c_str(), rename_to_on_close.c_str());
|
||||
if (ret != 0)
|
||||
LOGE("Failed to rename file %s -> %s.\n", rename_from_on_close.c_str(), rename_to_on_close.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
OSFilesystem::OSFilesystem(const std::string &base_)
|
||||
: base(base_)
|
||||
{
|
||||
#ifdef __linux__
|
||||
notify_fd = inotify_init1(IN_NONBLOCK);
|
||||
if (notify_fd < 0)
|
||||
LOGE("Failed to init inotify.\n");
|
||||
#else
|
||||
notify_fd = -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
OSFilesystem::~OSFilesystem()
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (notify_fd > 0)
|
||||
{
|
||||
for (auto &handler : handlers)
|
||||
inotify_rm_watch(notify_fd, handler.first);
|
||||
close(notify_fd);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
FileHandle OSFilesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
return MMapFile::open(Path::join(base, path), mode);
|
||||
}
|
||||
|
||||
std::string OSFilesystem::get_filesystem_path(const std::string &path)
|
||||
{
|
||||
return Path::join(base, path);
|
||||
}
|
||||
|
||||
int OSFilesystem::get_notification_fd() const
|
||||
{
|
||||
return notify_fd;
|
||||
}
|
||||
|
||||
void OSFilesystem::poll_notifications()
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (notify_fd < 0)
|
||||
return;
|
||||
|
||||
for (;;)
|
||||
{
|
||||
alignas(inotify_event) char buffer[sizeof(inotify_event) + NAME_MAX + 1];
|
||||
ssize_t ret = read(notify_fd, buffer, sizeof(buffer));
|
||||
|
||||
if (ret < 0)
|
||||
{
|
||||
if (errno != EAGAIN)
|
||||
LOGE("failed to read inotify fd.\n");
|
||||
break;
|
||||
}
|
||||
|
||||
struct inotify_event *current = nullptr;
|
||||
for (ssize_t i = 0; i < ret; i += current->len + sizeof(struct inotify_event))
|
||||
{
|
||||
current = reinterpret_cast<inotify_event *>(buffer + i);
|
||||
auto mask = current->mask;
|
||||
|
||||
int wd = current->wd;
|
||||
auto itr = handlers.find(wd);
|
||||
if (itr == end(handlers))
|
||||
continue;
|
||||
|
||||
FileNotifyType type;
|
||||
if (mask & IN_CLOSE_WRITE)
|
||||
type = FileNotifyType::FileChanged;
|
||||
else if (mask & (IN_CREATE | IN_MOVED_TO))
|
||||
type = FileNotifyType::FileCreated;
|
||||
else if (mask & (IN_DELETE | IN_DELETE_SELF | IN_MOVED_FROM))
|
||||
type = FileNotifyType::FileDeleted;
|
||||
else
|
||||
continue;
|
||||
|
||||
for (auto &func : itr->second.funcs)
|
||||
{
|
||||
if (func.func)
|
||||
{
|
||||
if (itr->second.directory)
|
||||
{
|
||||
auto notify_path = protocol + "://" + Path::join(func.path, current->name);
|
||||
func.func({ std::move(notify_path), type, func.virtual_handle });
|
||||
}
|
||||
else
|
||||
func.func({ protocol + "://" + func.path, type, func.virtual_handle });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void OSFilesystem::uninstall_notification(FileNotifyHandle handle)
|
||||
{
|
||||
#ifdef __linux__
|
||||
if (handle < 0)
|
||||
return;
|
||||
|
||||
if (notify_fd < 0)
|
||||
return;
|
||||
|
||||
//LOGI("Uninstalling notification: %d\n", handle);
|
||||
|
||||
auto real = virtual_to_real.find(handle);
|
||||
if (real == end(virtual_to_real))
|
||||
{
|
||||
LOGE("unknown virtual inotify handler.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
auto itr = handlers.find(static_cast<int>(real->second));
|
||||
if (itr == end(handlers))
|
||||
{
|
||||
LOGE("unknown inotify handler.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
auto handler_instance = find_if(begin(itr->second.funcs), end(itr->second.funcs), [=](const VirtualHandler &v) {
|
||||
return v.virtual_handle == handle;
|
||||
});
|
||||
|
||||
if (handler_instance == end(itr->second.funcs))
|
||||
{
|
||||
LOGE("unknown inotify handler path.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
itr->second.funcs.erase(handler_instance);
|
||||
|
||||
if (itr->second.funcs.empty())
|
||||
{
|
||||
inotify_rm_watch(notify_fd, real->second);
|
||||
handlers.erase(itr);
|
||||
}
|
||||
|
||||
virtual_to_real.erase(real);
|
||||
#else
|
||||
(void)handle;
|
||||
#endif
|
||||
}
|
||||
|
||||
FileNotifyHandle OSFilesystem::install_notification(const std::string &path,
|
||||
std::function<void (const FileNotifyInfo &)> func)
|
||||
{
|
||||
#ifdef __linux__
|
||||
//LOGI("Installing notification for: %s\n", path.c_str());
|
||||
if (notify_fd < 0)
|
||||
return -1;
|
||||
|
||||
FileStat s = {};
|
||||
if (!stat(path, s))
|
||||
{
|
||||
LOGE("inotify: path doesn't exist.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
auto resolved_path = Path::join(base, path);
|
||||
int wd = inotify_add_watch(notify_fd, resolved_path.c_str(),
|
||||
IN_MOVE | IN_CLOSE_WRITE | IN_CREATE | IN_DELETE | IN_DELETE_SELF);
|
||||
|
||||
if (wd < 0)
|
||||
{
|
||||
LOGE("Failed to create watch handle.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// We could have different paths which look different but resolve to the same wd, so handle that.
|
||||
auto itr = handlers.find(wd);
|
||||
if (itr == end(handlers))
|
||||
handlers[wd] = { {{ path, std::move(func), ++virtual_handle }}, s.type == PathType::Directory };
|
||||
else
|
||||
itr->second.funcs.push_back({ path, std::move(func), ++virtual_handle });
|
||||
|
||||
//LOGI(" Got handle: %d\n", virtual_handle);
|
||||
|
||||
virtual_to_real[virtual_handle] = wd;
|
||||
return static_cast<FileNotifyHandle>(virtual_handle);
|
||||
#else
|
||||
(void)path;
|
||||
(void)func;
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool OSFilesystem::remove(const std::string &path)
|
||||
{
|
||||
auto resolved_path = Path::join(base, path);
|
||||
return unlink(resolved_path.c_str()) == 0;
|
||||
}
|
||||
|
||||
bool OSFilesystem::move_yield(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto resolved_dst = Path::join(base, dst);
|
||||
auto resolved_src = Path::join(base, src);
|
||||
#if !defined(__linux__) || (defined(ANDROID) && (__ANDROID_API__ < __ANDROID_API_R__))
|
||||
// Workaround since Android does not have renameat2 until API level 30.
|
||||
// If we can exclusive create a new file, we can rename with replace somewhat safely.
|
||||
int fd = ::open(resolved_dst.c_str(), O_EXCL | O_RDWR | O_CREAT, 0600);
|
||||
if (fd >= 0)
|
||||
{
|
||||
::close(fd);
|
||||
return rename(resolved_src.c_str(), resolved_dst.c_str()) == 0;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
#else
|
||||
return renameat2(AT_FDCWD, resolved_src.c_str(), AT_FDCWD, resolved_dst.c_str(), RENAME_NOREPLACE) == 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool OSFilesystem::move_replace(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto resolved_dst = Path::join(base, dst);
|
||||
auto resolved_src = Path::join(base, src);
|
||||
return rename(resolved_src.c_str(), resolved_dst.c_str()) == 0;
|
||||
}
|
||||
|
||||
std::vector<ListEntry> OSFilesystem::list(const std::string &path)
|
||||
{
|
||||
auto directory = Path::join(base, path);
|
||||
DIR *dir = opendir(directory.c_str());
|
||||
if (!dir)
|
||||
{
|
||||
LOGE("Failed to open directory %s\n", path.c_str());
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<ListEntry> entries;
|
||||
struct dirent *entry;
|
||||
while ((entry = readdir(dir)))
|
||||
{
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
|
||||
auto joined_path = Path::join(path, entry->d_name);
|
||||
|
||||
PathType type;
|
||||
if (entry->d_type == DT_DIR)
|
||||
type = PathType::Directory;
|
||||
else if (entry->d_type == DT_REG)
|
||||
type = PathType::File;
|
||||
else if (entry->d_type != DT_UNKNOWN && entry->d_type != DT_LNK)
|
||||
type = PathType::Special;
|
||||
else
|
||||
{
|
||||
FileStat s;
|
||||
if (!stat(joined_path, s))
|
||||
{
|
||||
LOGE("Failed to stat file: %s\n", joined_path.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
type = s.type;
|
||||
}
|
||||
entries.push_back({ std::move(joined_path), type });
|
||||
}
|
||||
closedir(dir);
|
||||
return entries;
|
||||
}
|
||||
|
||||
bool OSFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto resolved_path = Path::join(base, path);
|
||||
struct STAT64 buf = {};
|
||||
if (::STAT64(resolved_path.c_str(), &buf) < 0)
|
||||
return false;
|
||||
|
||||
if (S_ISREG(buf.st_mode))
|
||||
stat.type = PathType::File;
|
||||
else if (S_ISDIR(buf.st_mode))
|
||||
stat.type = PathType::Directory;
|
||||
else
|
||||
stat.type = PathType::Special;
|
||||
|
||||
stat.size = uint64_t(buf.st_size);
|
||||
#ifdef __linux__
|
||||
stat.last_modified = buf.st_mtim.tv_sec * 1000000000ull + buf.st_mtim.tv_nsec;
|
||||
#else
|
||||
stat.last_modified = buf.st_mtimespec.tv_sec * 1000000000ull + buf.st_mtimespec.tv_nsec;
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "../filesystem.hpp"
|
||||
#include <unordered_map>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
class MMapFile final : public File
|
||||
{
|
||||
public:
|
||||
static FileHandle open(const std::string &path, FileMode mode);
|
||||
~MMapFile() override;
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t size) override;
|
||||
FileMappingHandle map_write(size_t map_size) override;
|
||||
void unmap(void *mapped, size_t size) override;
|
||||
uint64_t get_size() override;
|
||||
|
||||
private:
|
||||
bool init(const std::string &path, FileMode mode);
|
||||
bool query_stat();
|
||||
int fd = -1;
|
||||
size_t size = 0;
|
||||
bool has_write_map = false;
|
||||
std::string rename_from_on_close;
|
||||
std::string rename_to_on_close;
|
||||
|
||||
void report_error();
|
||||
};
|
||||
|
||||
class OSFilesystem : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
OSFilesystem(const std::string &base);
|
||||
~OSFilesystem();
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
FileHandle open(const std::string &path, FileMode mode) override;
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void (const FileNotifyInfo &)> func) override;
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
void poll_notifications() override;
|
||||
int get_notification_fd() const override;
|
||||
std::string get_filesystem_path(const std::string &path) override;
|
||||
|
||||
bool remove(const std::string &path) override;
|
||||
// Must implement atomic renaming semantics.
|
||||
// Atomically replaces dst.
|
||||
bool move_replace(const std::string &dst, const std::string &src) override;
|
||||
// If dst exists, nothing happens (and false is returned).
|
||||
bool move_yield(const std::string &dst, const std::string &src) override;
|
||||
|
||||
private:
|
||||
std::string base;
|
||||
|
||||
struct VirtualHandler
|
||||
{
|
||||
std::string path;
|
||||
std::function<void (const FileNotifyInfo &)> func;
|
||||
FileNotifyHandle virtual_handle;
|
||||
};
|
||||
|
||||
struct Handler
|
||||
{
|
||||
std::vector<VirtualHandler> funcs;
|
||||
bool directory;
|
||||
};
|
||||
std::unordered_map<FileNotifyHandle, Handler> handlers;
|
||||
std::unordered_map<FileNotifyHandle, FileNotifyHandle> virtual_to_real;
|
||||
int notify_fd;
|
||||
FileNotifyHandle virtual_handle = 0;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,895 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "fs-netfs.hpp"
|
||||
#include "path_utils.hpp"
|
||||
#include "logging.hpp"
|
||||
#include <assert.h>
|
||||
#include <queue>
|
||||
|
||||
#define HOST_IP "localhost"
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
struct FSNotifyCommand : LooperHandler
|
||||
{
|
||||
FSNotifyCommand(const string &protocol, unique_ptr<Socket> socket_)
|
||||
: LooperHandler(move(socket_)), expected(false)
|
||||
{
|
||||
reply_queue.emplace();
|
||||
auto &reply = reply_queue.back();
|
||||
reply.builder.add_u32(NETFS_NOTIFICATION);
|
||||
reply.builder.add_u32(NETFS_BEGIN_CHUNK_REQUEST);
|
||||
reply.builder.add_string(protocol);
|
||||
reply.writer.start(reply.builder.get_buffer());
|
||||
|
||||
result_reply.begin(4 * sizeof(uint32_t));
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
|
||||
state = NotificationLoop;
|
||||
}
|
||||
|
||||
void expected_destruction()
|
||||
{
|
||||
expected = true;
|
||||
}
|
||||
|
||||
~FSNotifyCommand()
|
||||
{
|
||||
if (!expected)
|
||||
terminate();
|
||||
}
|
||||
|
||||
void set_notify_cb(function<void (const FileNotifyInfo &)> func)
|
||||
{
|
||||
notify_cb = move(func);
|
||||
}
|
||||
|
||||
void push_register_notification(const string &path, promise<FileNotifyHandle> result)
|
||||
{
|
||||
if (reply_queue.empty() && socket->get_parent_looper())
|
||||
socket->get_parent_looper()->modify_handler(EVENT_IN | EVENT_OUT, *this);
|
||||
|
||||
reply_queue.emplace();
|
||||
auto &reply = reply_queue.back();
|
||||
reply.builder.add_u32(NETFS_REGISTER_NOTIFICATION);
|
||||
reply.builder.add_string(path);
|
||||
reply.writer.start(reply.builder.get_buffer());
|
||||
|
||||
replies.push(move(result));
|
||||
}
|
||||
|
||||
void push_unregister_notification(FileNotifyHandle handler, promise<FileNotifyHandle> result)
|
||||
{
|
||||
if (reply_queue.empty() && socket->get_parent_looper())
|
||||
socket->get_parent_looper()->modify_handler(EVENT_IN | EVENT_OUT, *this);
|
||||
|
||||
reply_queue.emplace();
|
||||
auto &reply = reply_queue.back();
|
||||
reply.builder.add_u32(NETFS_UNREGISTER_NOTIFICATION);
|
||||
reply.builder.add_u64(8);
|
||||
reply.builder.add_u64(uint64_t(handler));
|
||||
reply.writer.start(reply.builder.get_buffer());
|
||||
replies.push(move(result));
|
||||
}
|
||||
|
||||
void modify_looper(Looper &looper)
|
||||
{
|
||||
uint32_t mask = reply_queue.empty() ? EVENT_IN : (EVENT_IN | EVENT_OUT);
|
||||
looper.modify_handler(mask, *this);
|
||||
}
|
||||
|
||||
bool read_reply_data(Looper &looper)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
if (command_reader.complete())
|
||||
{
|
||||
if (last_cmd == NETFS_BEGIN_CHUNK_NOTIFICATION)
|
||||
{
|
||||
FileNotifyInfo info;
|
||||
info.path = result_reply.read_string();
|
||||
info.handle = FileNotifyHandle(result_reply.read_u64());
|
||||
auto type = result_reply.read_u32();
|
||||
switch (type)
|
||||
{
|
||||
case NETFS_FILE_CHANGED:
|
||||
info.type = FileNotifyType::FileChanged;
|
||||
break;
|
||||
case NETFS_FILE_DELETED:
|
||||
info.type = FileNotifyType::FileDeleted;
|
||||
break;
|
||||
case NETFS_FILE_CREATED:
|
||||
info.type = FileNotifyType::FileCreated;
|
||||
break;
|
||||
}
|
||||
|
||||
notify_cb(info);
|
||||
result_reply.begin(4 * sizeof(uint32_t));
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
modify_looper(looper);
|
||||
state = NotificationLoop;
|
||||
return true;
|
||||
}
|
||||
else if (last_cmd == NETFS_BEGIN_CHUNK_REPLY)
|
||||
{
|
||||
auto handle = int(result_reply.read_u64());
|
||||
|
||||
try
|
||||
{
|
||||
replies.front().set_value(handle);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
|
||||
assert(!replies.empty());
|
||||
replies.pop();
|
||||
|
||||
result_reply.begin(4 * sizeof(uint32_t));
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
modify_looper(looper);
|
||||
state = NotificationLoop;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
bool notification_loop(Looper &looper, EventFlags flags)
|
||||
{
|
||||
if (flags & EVENT_OUT)
|
||||
{
|
||||
if (reply_queue.empty())
|
||||
{
|
||||
looper.modify_handler(EVENT_IN, *this);
|
||||
return true;
|
||||
}
|
||||
|
||||
auto ret = reply_queue.front().writer.process(*socket);
|
||||
if (reply_queue.front().writer.complete())
|
||||
reply_queue.pop();
|
||||
|
||||
if (reply_queue.empty())
|
||||
{
|
||||
looper.modify_handler(EVENT_IN, *this);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
if (flags & EVENT_IN)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
if (command_reader.complete())
|
||||
{
|
||||
auto cmd = result_reply.read_u32();
|
||||
if (cmd == NETFS_BEGIN_CHUNK_NOTIFICATION || cmd == NETFS_BEGIN_CHUNK_REPLY)
|
||||
{
|
||||
if (result_reply.read_u32() != NETFS_ERROR_OK)
|
||||
return false;
|
||||
|
||||
last_cmd = cmd;
|
||||
auto size = result_reply.read_u64();
|
||||
|
||||
if (size)
|
||||
{
|
||||
// Either receive notification or acknowledgement.
|
||||
result_reply.begin(size);
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
state = ReadReplyData;
|
||||
looper.modify_handler(EVENT_IN, *this);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Acknowledge unregister notification.
|
||||
try
|
||||
{
|
||||
replies.front().set_value(0);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
|
||||
assert(!replies.empty());
|
||||
replies.pop();
|
||||
|
||||
result_reply.begin(4 * sizeof(uint32_t));
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
modify_looper(looper);
|
||||
state = NotificationLoop;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool handle(Looper &looper, EventFlags flags) override
|
||||
{
|
||||
if (state == ReadReplyData)
|
||||
return read_reply_data(looper);
|
||||
else if (state == NotificationLoop)
|
||||
return notification_loop(looper, flags);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
enum State
|
||||
{
|
||||
ReadReplyData,
|
||||
NotificationLoop
|
||||
};
|
||||
|
||||
State state = NotificationLoop;
|
||||
SocketReader command_reader;
|
||||
ReplyBuilder result_reply;
|
||||
uint32_t last_cmd = 0;
|
||||
|
||||
struct NotificationReply
|
||||
{
|
||||
SocketWriter writer;
|
||||
ReplyBuilder builder;
|
||||
};
|
||||
queue<NotificationReply> reply_queue;
|
||||
queue<promise<FileNotifyHandle>> replies;
|
||||
function<void (const FileNotifyInfo &info)> notify_cb;
|
||||
atomic_bool expected;
|
||||
};
|
||||
|
||||
struct FSReadCommand : LooperHandler
|
||||
{
|
||||
virtual ~FSReadCommand() = default;
|
||||
|
||||
FSReadCommand(const string &path, NetFSCommand command, unique_ptr<Socket> socket_)
|
||||
: LooperHandler(move(socket_))
|
||||
{
|
||||
reply_builder.begin();
|
||||
reply_builder.add_u32(command);
|
||||
reply_builder.add_u32(NETFS_BEGIN_CHUNK_REQUEST);
|
||||
reply_builder.add_string(path);
|
||||
command_writer.start(reply_builder.get_buffer());
|
||||
state = WriteCommand;
|
||||
}
|
||||
|
||||
bool write_command(Looper &looper)
|
||||
{
|
||||
auto ret = command_writer.process(*socket);
|
||||
if (command_writer.complete())
|
||||
{
|
||||
state = ReadReplySize;
|
||||
reply_builder.begin(4 * sizeof(uint32_t));
|
||||
command_reader.start(reply_builder.get_buffer());
|
||||
looper.modify_handler(EVENT_IN, *this);
|
||||
return true;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
bool read_reply_size(Looper &)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
if (command_reader.complete())
|
||||
{
|
||||
if (reply_builder.read_u32() != NETFS_BEGIN_CHUNK_REPLY)
|
||||
return false;
|
||||
|
||||
if (reply_builder.read_u32() != NETFS_ERROR_OK)
|
||||
return false;
|
||||
|
||||
uint64_t reply_size = reply_builder.read_u64();
|
||||
if (reply_size == 0)
|
||||
return false;
|
||||
|
||||
reply_builder.begin(reply_size);
|
||||
command_reader.start(reply_builder.get_buffer());
|
||||
state = ReadReply;
|
||||
return true;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
bool read_reply(Looper &)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
if (command_reader.complete())
|
||||
{
|
||||
parse_reply();
|
||||
return false;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
bool handle(Looper &looper, EventFlags) override
|
||||
{
|
||||
if (state == WriteCommand)
|
||||
return write_command(looper);
|
||||
else if (state == ReadReplySize)
|
||||
return read_reply_size(looper);
|
||||
else if (state == ReadReply)
|
||||
return read_reply(looper);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
enum State
|
||||
{
|
||||
WriteCommand,
|
||||
ReadReplySize,
|
||||
ReadReply
|
||||
};
|
||||
State state = WriteCommand;
|
||||
SocketReader command_reader;
|
||||
SocketWriter command_writer;
|
||||
ReplyBuilder reply_builder;
|
||||
|
||||
virtual void parse_reply() = 0;
|
||||
};
|
||||
|
||||
struct FSReader : FSReadCommand
|
||||
{
|
||||
FSReader(const string &path, unique_ptr<Socket> socket_)
|
||||
: FSReadCommand(path, NETFS_READ_FILE, move(socket_))
|
||||
{
|
||||
}
|
||||
|
||||
~FSReader()
|
||||
{
|
||||
if (!got_reply)
|
||||
result.set_exception(make_exception_ptr(runtime_error("file read")));
|
||||
}
|
||||
|
||||
void parse_reply() override
|
||||
{
|
||||
got_reply = true;
|
||||
try
|
||||
{
|
||||
result.set_value(reply_builder.consume_buffer());
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
promise<vector<uint8_t>> result;
|
||||
bool got_reply = false;
|
||||
};
|
||||
|
||||
struct FSList : FSReadCommand
|
||||
{
|
||||
FSList(const string &path, unique_ptr<Socket> socket_)
|
||||
: FSReadCommand(path, NETFS_LIST, move(socket_))
|
||||
{
|
||||
}
|
||||
|
||||
~FSList()
|
||||
{
|
||||
if (!got_reply)
|
||||
result.set_exception(make_exception_ptr(runtime_error("List failed")));
|
||||
}
|
||||
|
||||
void parse_reply() override
|
||||
{
|
||||
uint32_t entries = reply_builder.read_u32();
|
||||
vector<ListEntry> list;
|
||||
for (uint32_t i = 0; i < entries; i++)
|
||||
{
|
||||
auto path = reply_builder.read_string();
|
||||
auto type = reply_builder.read_u32();
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case NETFS_FILE_TYPE_PLAIN:
|
||||
list.push_back({ move(path), PathType::File });
|
||||
break;
|
||||
case NETFS_FILE_TYPE_DIRECTORY:
|
||||
list.push_back({ move(path), PathType::Directory });
|
||||
break;
|
||||
case NETFS_FILE_TYPE_SPECIAL:
|
||||
list.push_back({ move(path), PathType::Special });
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
got_reply = true;
|
||||
try
|
||||
{
|
||||
result.set_value(move(list));
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
promise<vector<ListEntry>> result;
|
||||
bool got_reply = false;
|
||||
};
|
||||
|
||||
struct FSStat : FSReadCommand
|
||||
{
|
||||
FSStat(const string &path, unique_ptr<Socket> socket_)
|
||||
: FSReadCommand(path, NETFS_STAT, move(socket_))
|
||||
{
|
||||
}
|
||||
|
||||
~FSStat()
|
||||
{
|
||||
// Throw exception instead in calling thread.
|
||||
if (!got_reply)
|
||||
result.set_exception(make_exception_ptr(runtime_error("Failed stat")));
|
||||
}
|
||||
|
||||
void parse_reply() override
|
||||
{
|
||||
uint64_t size = reply_builder.read_u64();
|
||||
uint32_t type = reply_builder.read_u32();
|
||||
uint64_t last_modified = reply_builder.read_u64();
|
||||
FileStat s;
|
||||
s.size = size;
|
||||
s.last_modified = last_modified;
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case NETFS_FILE_TYPE_PLAIN:
|
||||
s.type = PathType::File;
|
||||
break;
|
||||
case NETFS_FILE_TYPE_DIRECTORY:
|
||||
s.type = PathType::Directory;
|
||||
break;
|
||||
case NETFS_FILE_TYPE_SPECIAL:
|
||||
s.type = PathType::Special;
|
||||
break;
|
||||
}
|
||||
|
||||
got_reply = true;
|
||||
try
|
||||
{
|
||||
result.set_value(s);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
std::promise<FileStat> result;
|
||||
bool got_reply = false;
|
||||
};
|
||||
|
||||
struct FSWriteCommand : LooperHandler
|
||||
{
|
||||
FSWriteCommand(const string &path, const vector<uint8_t> &buffer, unique_ptr<Socket> socket_)
|
||||
: LooperHandler(move(socket_))
|
||||
{
|
||||
target_size = buffer.size();
|
||||
|
||||
reply_builder.begin();
|
||||
result_reply.begin(4 * sizeof(uint32_t));
|
||||
|
||||
reply_builder.add_u32(NETFS_WRITE_FILE);
|
||||
reply_builder.add_u32(NETFS_BEGIN_CHUNK_REQUEST);
|
||||
reply_builder.add_string(path);
|
||||
reply_builder.add_u32(NETFS_BEGIN_CHUNK_REQUEST);
|
||||
reply_builder.add_u64(buffer.size());
|
||||
reply_builder.add_buffer(buffer);
|
||||
command_writer.start(reply_builder.get_buffer());
|
||||
command_reader.start(result_reply.get_buffer());
|
||||
state = WriteCommand;
|
||||
}
|
||||
|
||||
bool write_command(Looper &looper, EventFlags flags)
|
||||
{
|
||||
if (flags & EVENT_IN)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
// Received message before we completed the write, must be an error.
|
||||
if (command_reader.complete())
|
||||
return false;
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
else if (flags & EVENT_OUT)
|
||||
{
|
||||
auto ret = command_writer.process(*socket);
|
||||
if (command_writer.complete())
|
||||
{
|
||||
// Done writing, wait for reply.
|
||||
looper.modify_handler(EVENT_IN, *this);
|
||||
state = ReadReply;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
~FSWriteCommand()
|
||||
{
|
||||
if (!got_reply)
|
||||
result.set_exception(make_exception_ptr(runtime_error("Failed write")));
|
||||
}
|
||||
|
||||
bool read_reply(Looper &)
|
||||
{
|
||||
auto ret = command_reader.process(*socket);
|
||||
if (command_reader.complete())
|
||||
{
|
||||
if (result_reply.read_u32() != NETFS_BEGIN_CHUNK_REPLY)
|
||||
return false;
|
||||
if (result_reply.read_u32() != NETFS_ERROR_OK)
|
||||
return false;
|
||||
if (result_reply.read_u64() != target_size)
|
||||
return false;
|
||||
|
||||
got_reply = true;
|
||||
try
|
||||
{
|
||||
result.set_value(NETFS_ERROR_OK);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return (ret > 0) || (ret == Socket::ErrorWouldBlock);
|
||||
}
|
||||
|
||||
bool handle(Looper &looper, EventFlags flags) override
|
||||
{
|
||||
if (state == WriteCommand)
|
||||
return write_command(looper, flags);
|
||||
else if (state == ReadReply)
|
||||
return read_reply(looper);
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
enum State
|
||||
{
|
||||
WriteCommand,
|
||||
ReadReply
|
||||
};
|
||||
|
||||
State state = WriteCommand;
|
||||
SocketReader command_reader;
|
||||
SocketWriter command_writer;
|
||||
ReplyBuilder reply_builder;
|
||||
ReplyBuilder result_reply;
|
||||
size_t target_size = 0;
|
||||
|
||||
promise<NetFSError> result;
|
||||
bool got_reply = false;
|
||||
};
|
||||
|
||||
NetworkFilesystem::NetworkFilesystem()
|
||||
{
|
||||
looper_thread = thread(&NetworkFilesystem::looper_entry, this);
|
||||
}
|
||||
|
||||
void NetworkFilesystem::looper_entry()
|
||||
{
|
||||
while (looper.wait_idle(-1) >= 0);
|
||||
}
|
||||
|
||||
void NetworkFilesystem::setup_notification()
|
||||
{
|
||||
auto socket = Socket::connect(HOST_IP, 7070);
|
||||
if (!socket)
|
||||
return;
|
||||
notify = new FSNotifyCommand(protocol, move(socket));
|
||||
notify->set_notify_cb([this](const FileNotifyInfo &info) {
|
||||
signal_notification(info);
|
||||
});
|
||||
|
||||
// Move capture would be nice ...
|
||||
looper.run_in_looper([this]() {
|
||||
looper.register_handler(EVENT_OUT, unique_ptr<FSNotifyCommand>(notify));
|
||||
});
|
||||
}
|
||||
|
||||
void NetworkFilesystem::uninstall_notification(FileNotifyHandle handle)
|
||||
{
|
||||
if (!notify)
|
||||
setup_notification();
|
||||
if (!notify)
|
||||
return;
|
||||
|
||||
auto itr = handlers.find(handle);
|
||||
if (itr == end(handlers))
|
||||
return;
|
||||
handlers.erase(itr);
|
||||
|
||||
auto *value = new promise<FileNotifyHandle>;
|
||||
auto result = value->get_future();
|
||||
looper.run_in_looper([this, value, handle]() {
|
||||
notify->push_unregister_notification(handle, move(*value));
|
||||
delete value;
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
result.wait();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
void NetworkFilesystem::signal_notification(const FileNotifyInfo &info)
|
||||
{
|
||||
lock_guard<mutex> holder{lock};
|
||||
pending.push_back(info);
|
||||
}
|
||||
|
||||
void NetworkFilesystem::poll_notifications()
|
||||
{
|
||||
vector<FileNotifyInfo> tmp_pending;
|
||||
{
|
||||
lock_guard<mutex> holder{lock};
|
||||
swap(tmp_pending, pending);
|
||||
}
|
||||
|
||||
for (auto ¬ification : tmp_pending)
|
||||
{
|
||||
auto &func = handlers[notification.handle];
|
||||
if (func)
|
||||
func(notification);
|
||||
}
|
||||
}
|
||||
|
||||
FileNotifyHandle NetworkFilesystem::install_notification(const std::string &path,
|
||||
std::function<void(const FileNotifyInfo &)> func)
|
||||
{
|
||||
if (!notify)
|
||||
setup_notification();
|
||||
if (!notify)
|
||||
return -1;
|
||||
|
||||
auto *value = new promise<FileNotifyHandle>;
|
||||
auto result = value->get_future();
|
||||
|
||||
looper.run_in_looper([this, value, path]() {
|
||||
notify->push_register_notification(path, move(*value));
|
||||
delete value;
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
auto handle = result.get();
|
||||
handlers[handle] = move(func);
|
||||
return handle;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
vector<ListEntry> NetworkFilesystem::list(const std::string &path)
|
||||
{
|
||||
auto joined = protocol + "://" + path;
|
||||
auto socket = Socket::connect(HOST_IP, 7070);
|
||||
if (!socket)
|
||||
return {};
|
||||
|
||||
unique_ptr<FSList> handler(new FSList(joined, move(socket)));
|
||||
auto fut = handler->result.get_future();
|
||||
|
||||
looper.run_in_looper([&]() {
|
||||
looper.register_handler(EVENT_OUT, move(handler));
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
return fut.get();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
NetworkFile::~NetworkFile()
|
||||
{
|
||||
unmap();
|
||||
}
|
||||
|
||||
NetworkFile *NetworkFile::open(Granite::Looper &looper, const std::string &path, Granite::FileMode mode)
|
||||
{
|
||||
auto *file = new NetworkFile;
|
||||
if (!file->init(looper, path, mode))
|
||||
{
|
||||
delete file;
|
||||
return nullptr;
|
||||
}
|
||||
else
|
||||
return file;
|
||||
}
|
||||
|
||||
bool NetworkFile::init(Looper &looper_, const std::string &path_, FileMode mode_)
|
||||
{
|
||||
path = path_;
|
||||
mode = mode_;
|
||||
looper = &looper_;
|
||||
|
||||
if (mode == FileMode::ReadWrite)
|
||||
{
|
||||
LOGE("Unsupported file mode.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (mode == FileMode::ReadOnly)
|
||||
{
|
||||
if (!reopen())
|
||||
{
|
||||
LOGE("Failed to connect to server.\n");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void NetworkFile::unmap()
|
||||
{
|
||||
if (mode == FileMode::WriteOnly && has_buffer && need_flush)
|
||||
{
|
||||
need_flush = false;
|
||||
auto socket = Socket::connect(HOST_IP, 7070);
|
||||
if (!socket)
|
||||
throw runtime_error("Failed to connect to server.");
|
||||
|
||||
auto handler = unique_ptr<FSWriteCommand>(new FSWriteCommand(path, buffer, move(socket)));
|
||||
auto reply = handler->result.get_future();
|
||||
looper->run_in_looper([&handler, this]() {
|
||||
looper->register_handler(EVENT_OUT | EVENT_IN, move(handler));
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
NetFSError error = reply.get();
|
||||
if (error != NETFS_ERROR_OK)
|
||||
LOGE("Failed to write file: %s\n", path.c_str());
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
LOGE("Failed to write file: %s\n", path.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool NetworkFile::reopen()
|
||||
{
|
||||
if (mode == FileMode::ReadOnly)
|
||||
{
|
||||
has_buffer = false;
|
||||
auto socket = Socket::connect(HOST_IP, 7070);
|
||||
if (!socket)
|
||||
return false;
|
||||
|
||||
auto *handler = new FSReader(path, move(socket));
|
||||
future = handler->result.get_future();
|
||||
|
||||
// Capture-by-move would be nice here.
|
||||
looper->run_in_looper([handler, this]() {
|
||||
looper->register_handler(EVENT_OUT, unique_ptr<FSReader>(handler));
|
||||
});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void *NetworkFile::map_write(size_t size)
|
||||
{
|
||||
has_buffer = true;
|
||||
need_flush = true;
|
||||
buffer.resize(size);
|
||||
return buffer.empty() ? nullptr : buffer.data();
|
||||
}
|
||||
|
||||
void *NetworkFile::map()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!has_buffer)
|
||||
{
|
||||
buffer = future.get();
|
||||
has_buffer = true;
|
||||
}
|
||||
return buffer.empty() ? nullptr : buffer.data();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
size_t NetworkFile::get_size()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!has_buffer)
|
||||
{
|
||||
buffer = future.get();
|
||||
has_buffer = true;
|
||||
}
|
||||
return buffer.size();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
unique_ptr<File> NetworkFilesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
auto joined = protocol + "://" + path;
|
||||
return unique_ptr<File>(NetworkFile::open(looper, move(joined), mode));
|
||||
}
|
||||
|
||||
bool NetworkFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto joined = protocol + "://" + path;
|
||||
auto socket = Socket::connect(HOST_IP, 7070);
|
||||
if (!socket)
|
||||
return false;
|
||||
|
||||
unique_ptr<FSStat> handler(new FSStat(joined, move(socket)));
|
||||
auto fut = handler->result.get_future();
|
||||
|
||||
looper.run_in_looper([&]() {
|
||||
looper.register_handler(EVENT_OUT, move(handler));
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
stat = fut.get();
|
||||
return true;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
NetworkFilesystem::~NetworkFilesystem()
|
||||
{
|
||||
if (notify)
|
||||
notify->expected_destruction();
|
||||
|
||||
looper.kill();
|
||||
if (looper_thread.joinable())
|
||||
looper_thread.join();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "fs-netfs.hpp"
|
||||
#include "network.hpp"
|
||||
#include "../filesystem.hpp"
|
||||
#include "netfs.hpp"
|
||||
#include <unordered_map>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
struct FSReader;
|
||||
class NetworkFile : public File
|
||||
{
|
||||
public:
|
||||
static NetworkFile *open(Looper &looper, const std::string &path, FileMode mode);
|
||||
~NetworkFile();
|
||||
void *map() override;
|
||||
void *map_write(size_t size) override;
|
||||
void unmap() override;
|
||||
size_t get_size() override;
|
||||
bool reopen() override;
|
||||
|
||||
private:
|
||||
NetworkFile() = default;
|
||||
bool init(Looper &looper, const std::string &path, FileMode mode);
|
||||
std::string path;
|
||||
FileMode mode;
|
||||
Looper *looper = nullptr;
|
||||
std::future<std::vector<uint8_t>> future;
|
||||
std::vector<uint8_t> buffer;
|
||||
bool has_buffer = false;
|
||||
bool need_flush = false;
|
||||
};
|
||||
|
||||
struct FSNotifyCommand;
|
||||
class NetworkFilesystem : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
NetworkFilesystem();
|
||||
~NetworkFilesystem();
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
std::unique_ptr<File> open(const std::string &path, FileMode mode) override;
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void (const FileNotifyInfo &)> func) override;
|
||||
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
|
||||
void poll_notifications() override;
|
||||
|
||||
int get_notification_fd() const override
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
private:
|
||||
std::thread looper_thread;
|
||||
Looper looper;
|
||||
void looper_entry();
|
||||
FSNotifyCommand *notify = nullptr;
|
||||
|
||||
std::unordered_map<FileNotifyHandle, std::function<void (const FileNotifyInfo &)>> handlers;
|
||||
std::mutex lock;
|
||||
std::vector<FileNotifyInfo> pending;
|
||||
|
||||
void setup_notification();
|
||||
void signal_notification(const FileNotifyInfo &info);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "path_utils.hpp"
|
||||
#include "filesystem.hpp"
|
||||
#include "intrusive.hpp"
|
||||
#include "logging.hpp"
|
||||
#include <string>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
template <typename T>
|
||||
class VolatileSource : public Util::IntrusivePtrEnabled<VolatileSource<T>>
|
||||
{
|
||||
public:
|
||||
VolatileSource(Filesystem *fs_, const std::string &path_)
|
||||
: fs(fs_), path(Path::enforce_protocol(path_))
|
||||
{
|
||||
}
|
||||
|
||||
VolatileSource() = default;
|
||||
|
||||
~VolatileSource()
|
||||
{
|
||||
deinit();
|
||||
}
|
||||
|
||||
protected:
|
||||
Filesystem *fs = nullptr;
|
||||
std::string path;
|
||||
void deinit()
|
||||
{
|
||||
if (notify_backend && notify_handle >= 0)
|
||||
notify_backend->uninstall_notification(notify_handle);
|
||||
notify_backend = nullptr;
|
||||
notify_handle = -1;
|
||||
}
|
||||
|
||||
bool init()
|
||||
{
|
||||
if (path.empty() || !fs)
|
||||
return false;
|
||||
|
||||
auto file = fs->open_readonly_mapping(path);
|
||||
if (!file)
|
||||
{
|
||||
LOGE("Failed to open volatile file: %s\n", path.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto *self = static_cast<T *>(this);
|
||||
self->update(std::move(file));
|
||||
|
||||
auto paths = Path::protocol_split(path);
|
||||
auto *proto = fs->get_backend(paths.first);
|
||||
if (proto)
|
||||
{
|
||||
// Listen to directory so we can track file moves properly.
|
||||
notify_handle = proto->install_notification(Path::basedir(paths.second), [&](const FileNotifyInfo &info) {
|
||||
if (info.type == FileNotifyType::FileDeleted)
|
||||
return;
|
||||
if (info.path != path)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
auto f = fs->open_readonly_mapping(info.path);
|
||||
if (!f)
|
||||
return;
|
||||
auto *s = static_cast<T *>(this);
|
||||
s->update(std::move(f));
|
||||
}
|
||||
catch (const std::exception &e)
|
||||
{
|
||||
LOGE("Caught update exception: %s\n", e.what());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
FileNotifyHandle notify_handle = -1;
|
||||
FilesystemBackend *notify_backend = nullptr;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
using VolatileHandle = Util::IntrusivePtr<VolatileSource<T>>;
|
||||
}
|
||||
+471
@@ -0,0 +1,471 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include "os_filesystem.hpp"
|
||||
#include "path_utils.hpp"
|
||||
#include "logging.hpp"
|
||||
#include <stdexcept>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <atomic>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
static bool ensure_directory_inner(const std::string &path)
|
||||
{
|
||||
if (Path::is_root_path(path))
|
||||
return false;
|
||||
|
||||
auto wpath = Path::to_utf16(path);
|
||||
|
||||
struct __stat64 s;
|
||||
if (::_wstat64(wpath.c_str(), &s) >= 0 && (s.st_mode & _S_IFDIR) != 0)
|
||||
return true;
|
||||
|
||||
auto basedir = Path::basedir(path);
|
||||
if (!ensure_directory_inner(basedir))
|
||||
return false;
|
||||
|
||||
if (!CreateDirectoryW(wpath.c_str(), nullptr))
|
||||
return GetLastError() == ERROR_ALREADY_EXISTS;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ensure_directory(const std::string &path)
|
||||
{
|
||||
auto basedir = Path::basedir(path);
|
||||
return ensure_directory_inner(basedir);
|
||||
}
|
||||
|
||||
FileHandle MappedFile::open(const std::string &path, Granite::FileMode mode)
|
||||
{
|
||||
auto file = Util::make_handle<MappedFile>();
|
||||
if (!file->init(path, mode))
|
||||
file.reset();
|
||||
return file;
|
||||
}
|
||||
|
||||
static std::atomic_uint32_t global_transaction_counter;
|
||||
|
||||
bool MappedFile::init(const std::string &path, FileMode mode)
|
||||
{
|
||||
DWORD access = 0;
|
||||
DWORD disposition = 0;
|
||||
|
||||
switch (mode)
|
||||
{
|
||||
case FileMode::ReadOnly:
|
||||
access = GENERIC_READ;
|
||||
disposition = OPEN_EXISTING;
|
||||
break;
|
||||
|
||||
case FileMode::ReadWrite:
|
||||
if (!ensure_directory(path))
|
||||
{
|
||||
LOGE("MappedFile failed to create directory.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
access = GENERIC_READ | GENERIC_WRITE;
|
||||
disposition = OPEN_ALWAYS;
|
||||
break;
|
||||
|
||||
case FileMode::WriteOnly:
|
||||
case FileMode::WriteOnlyTransactional:
|
||||
if (!ensure_directory(path))
|
||||
{
|
||||
LOGE("MappedFile failed to create directory.\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
access = GENERIC_READ | GENERIC_WRITE;
|
||||
disposition = CREATE_ALWAYS;
|
||||
break;
|
||||
}
|
||||
|
||||
if (mode == FileMode::WriteOnlyTransactional)
|
||||
{
|
||||
// Use atomic file rename to ensure that a file is written atomically.
|
||||
rename_to_on_close = path;
|
||||
rename_from_on_close =
|
||||
path + ".tmp." +
|
||||
std::to_string(GetCurrentProcessId()) + "." +
|
||||
std::to_string(global_transaction_counter.fetch_add(1, std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
auto wpath = Path::to_utf16(rename_from_on_close.empty() ? path : rename_from_on_close);
|
||||
|
||||
file = CreateFileW(wpath.c_str(), access, FILE_SHARE_READ, nullptr, disposition,
|
||||
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, INVALID_HANDLE_VALUE);
|
||||
if (file == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
rename_to_on_close.clear();
|
||||
rename_from_on_close.clear();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (mode != FileMode::WriteOnly && mode != FileMode::WriteOnlyTransactional)
|
||||
{
|
||||
DWORD hi;
|
||||
DWORD lo = GetFileSize(file, &hi);
|
||||
size = (uint64_t(hi) << 32) | uint32_t(lo);
|
||||
file_mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t MappedFile::get_size()
|
||||
{
|
||||
return size;
|
||||
}
|
||||
|
||||
struct PageSizeQuery
|
||||
{
|
||||
PageSizeQuery()
|
||||
{
|
||||
SYSTEM_INFO system_info = {};
|
||||
GetSystemInfo(&system_info);
|
||||
page_size = system_info.dwPageSize;
|
||||
}
|
||||
uint32_t page_size = 0;
|
||||
};
|
||||
static PageSizeQuery static_page_size_query;
|
||||
|
||||
FileMappingHandle MappedFile::map_subset(uint64_t offset, size_t range)
|
||||
{
|
||||
if (offset + range > size)
|
||||
return {};
|
||||
|
||||
if (!file_mapping)
|
||||
return {};
|
||||
|
||||
uint64_t begin_map = offset & ~uint64_t(static_page_size_query.page_size - 1);
|
||||
|
||||
DWORD hi = DWORD(begin_map >> 32);
|
||||
DWORD lo = DWORD(begin_map & 0xffffffffu);
|
||||
uint64_t end_mapping = offset + range;
|
||||
size_t mapped_size = end_mapping - begin_map;
|
||||
|
||||
void *mapped = MapViewOfFile(file_mapping, FILE_MAP_READ, hi, lo, mapped_size);
|
||||
if (!mapped)
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileMapping>(
|
||||
reference_from_this(), offset,
|
||||
static_cast<uint8_t *>(mapped) + begin_map, mapped_size,
|
||||
offset - begin_map, range);
|
||||
}
|
||||
|
||||
FileMappingHandle MappedFile::map_write(size_t map_size)
|
||||
{
|
||||
size = map_size;
|
||||
|
||||
#ifdef _WIN64
|
||||
DWORD hi = DWORD(size >> 32);
|
||||
DWORD lo = DWORD(size & 0xffffffffu);
|
||||
#else
|
||||
DWORD hi = 0;
|
||||
DWORD lo = DWORD(size);
|
||||
#endif
|
||||
|
||||
HANDLE file_view = CreateFileMappingW(file, nullptr, PAGE_READWRITE, hi, lo, nullptr);
|
||||
if (!file_view)
|
||||
return {};
|
||||
|
||||
void *mapped = MapViewOfFile(file_view, FILE_MAP_ALL_ACCESS, 0, 0, size);
|
||||
CloseHandle(file_view);
|
||||
|
||||
if (!mapped)
|
||||
return {};
|
||||
|
||||
return Util::make_handle<FileMapping>(reference_from_this(), 0,
|
||||
static_cast<uint8_t *>(mapped), size,
|
||||
0, size);
|
||||
}
|
||||
|
||||
void MappedFile::unmap(void *mapped, size_t)
|
||||
{
|
||||
if (mapped)
|
||||
UnmapViewOfFile(mapped);
|
||||
}
|
||||
|
||||
MappedFile::~MappedFile()
|
||||
{
|
||||
if (file_mapping)
|
||||
CloseHandle(file_mapping);
|
||||
if (file != INVALID_HANDLE_VALUE)
|
||||
CloseHandle(file);
|
||||
|
||||
if (!rename_from_on_close.empty() && !rename_to_on_close.empty())
|
||||
{
|
||||
auto to_w16 = Path::to_utf16(rename_to_on_close);
|
||||
auto from_w16 = Path::to_utf16(rename_from_on_close);
|
||||
DWORD code = S_OK;
|
||||
|
||||
if (!MoveFileW(from_w16.c_str(), to_w16.c_str()))
|
||||
{
|
||||
code = GetLastError();
|
||||
if (code == ERROR_ALREADY_EXISTS && !ReplaceFileW(to_w16.c_str(), from_w16.c_str(), nullptr, 0, nullptr, nullptr))
|
||||
code = GetLastError();
|
||||
}
|
||||
|
||||
if (FAILED(code))
|
||||
LOGE("Failed to rename file %s -> %s (0x%lx).\n", rename_from_on_close.c_str(), rename_to_on_close.c_str(), code);
|
||||
}
|
||||
}
|
||||
|
||||
OSFilesystem::OSFilesystem(const std::string &base_)
|
||||
: base(base_)
|
||||
{
|
||||
}
|
||||
|
||||
OSFilesystem::~OSFilesystem()
|
||||
{
|
||||
for (auto &handler : handlers)
|
||||
{
|
||||
CancelIo(handler.second.handle);
|
||||
CloseHandle(handler.second.handle);
|
||||
CloseHandle(handler.second.event);
|
||||
}
|
||||
}
|
||||
|
||||
std::string OSFilesystem::get_filesystem_path(const std::string &path)
|
||||
{
|
||||
return Path::join(base, path);
|
||||
}
|
||||
|
||||
FileHandle OSFilesystem::open(const std::string &path, FileMode mode)
|
||||
{
|
||||
return MappedFile::open(Path::join(base, path), mode);
|
||||
}
|
||||
|
||||
void OSFilesystem::poll_notifications()
|
||||
{
|
||||
for (auto &handler : handlers)
|
||||
{
|
||||
if (WaitForSingleObject(handler.second.event, 0) != WAIT_OBJECT_0)
|
||||
continue;
|
||||
|
||||
DWORD bytes_returned;
|
||||
if (!GetOverlappedResult(handler.second.handle, &handler.second.overlapped, &bytes_returned, TRUE))
|
||||
continue;
|
||||
|
||||
size_t offset = 0;
|
||||
const FILE_NOTIFY_INFORMATION *info = nullptr;
|
||||
do
|
||||
{
|
||||
info = reinterpret_cast<const FILE_NOTIFY_INFORMATION *>(
|
||||
reinterpret_cast<const uint8_t *>(handler.second.async_buffer) + offset);
|
||||
|
||||
FileNotifyInfo notify;
|
||||
notify.handle = handler.first;
|
||||
notify.path = Path::join(handler.second.path,
|
||||
Path::to_utf8(info->FileName,
|
||||
info->FileNameLength / sizeof(wchar_t)));
|
||||
|
||||
switch (info->Action)
|
||||
{
|
||||
case FILE_ACTION_ADDED:
|
||||
case FILE_ACTION_RENAMED_NEW_NAME:
|
||||
notify.type = FileNotifyType::FileCreated;
|
||||
if (handler.second.func)
|
||||
handler.second.func(notify);
|
||||
break;
|
||||
|
||||
case FILE_ACTION_REMOVED:
|
||||
case FILE_ACTION_RENAMED_OLD_NAME:
|
||||
notify.type = FileNotifyType::FileDeleted;
|
||||
if (handler.second.func)
|
||||
handler.second.func(notify);
|
||||
break;
|
||||
|
||||
case FILE_ACTION_MODIFIED:
|
||||
notify.type = FileNotifyType::FileChanged;
|
||||
if (handler.second.func)
|
||||
handler.second.func(notify);
|
||||
break;
|
||||
|
||||
default:
|
||||
LOGE("Invalid notify type.\n");
|
||||
break;
|
||||
}
|
||||
|
||||
offset += info->NextEntryOffset;
|
||||
} while (info->NextEntryOffset != 0);
|
||||
|
||||
kick_async(handler.second);
|
||||
}
|
||||
}
|
||||
|
||||
void OSFilesystem::uninstall_notification(FileNotifyHandle id)
|
||||
{
|
||||
auto itr = handlers.find(id);
|
||||
if (itr != end(handlers))
|
||||
{
|
||||
CancelIo(itr->second.handle);
|
||||
CloseHandle(itr->second.handle);
|
||||
CloseHandle(itr->second.event);
|
||||
handlers.erase(itr);
|
||||
}
|
||||
}
|
||||
|
||||
void OSFilesystem::kick_async(Handler &handler)
|
||||
{
|
||||
handler.overlapped = {};
|
||||
handler.overlapped.hEvent = handler.event;
|
||||
|
||||
auto ret = ReadDirectoryChangesW(handler.handle, handler.async_buffer, sizeof(handler.async_buffer), FALSE,
|
||||
FILE_NOTIFY_CHANGE_LAST_WRITE | FILE_NOTIFY_CHANGE_CREATION | FILE_NOTIFY_CHANGE_FILE_NAME,
|
||||
nullptr, &handler.overlapped, nullptr);
|
||||
|
||||
if (!ret && GetLastError() != ERROR_IO_PENDING)
|
||||
{
|
||||
LOGE("Failed to read directory changes async.\n");
|
||||
}
|
||||
}
|
||||
|
||||
FileNotifyHandle OSFilesystem::install_notification(const std::string &path, std::function<void(const FileNotifyInfo &)> func)
|
||||
{
|
||||
FileStat s = {};
|
||||
if (!stat(path, s))
|
||||
{
|
||||
LOGE("Window inotify: path doesn't exist.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (s.type != PathType::Directory)
|
||||
{
|
||||
LOGE("Windows inotify: Implementation only supports directories.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
auto resolved_path = Path::to_utf16(Path::join(base, path));
|
||||
HANDLE handle =
|
||||
CreateFileW(resolved_path.c_str(), FILE_LIST_DIRECTORY, FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
|
||||
nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, nullptr);
|
||||
if (handle == INVALID_HANDLE_VALUE)
|
||||
{
|
||||
LOGE("Failed to open directory for watching.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
HANDLE event = CreateEvent(nullptr, FALSE, FALSE, nullptr);
|
||||
if (event == nullptr)
|
||||
{
|
||||
CloseHandle(handle);
|
||||
return -1;
|
||||
}
|
||||
|
||||
handle_id++;
|
||||
Handler handler;
|
||||
handler.path = protocol + "://" + path;
|
||||
handler.func = std::move(func);
|
||||
handler.handle = handle;
|
||||
handler.event = event;
|
||||
auto &h = handlers[handle_id];
|
||||
h = std::move(handler);
|
||||
kick_async(h);
|
||||
|
||||
return handle_id;
|
||||
}
|
||||
|
||||
bool OSFilesystem::remove(const std::string &path)
|
||||
{
|
||||
auto joined = Path::to_utf16(Path::join(base, path));
|
||||
return bool(DeleteFileW(joined.c_str()));
|
||||
}
|
||||
|
||||
bool OSFilesystem::move_yield(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto joined_dst = Path::to_utf16(Path::join(base, dst));
|
||||
auto joined_src = Path::to_utf16(Path::join(base, src));
|
||||
return bool(MoveFileW(joined_src.c_str(), joined_dst.c_str()));
|
||||
}
|
||||
|
||||
bool OSFilesystem::move_replace(const std::string &dst, const std::string &src)
|
||||
{
|
||||
auto joined_dst = Path::to_utf16(Path::join(base, dst));
|
||||
auto joined_src = Path::to_utf16(Path::join(base, src));
|
||||
if (MoveFileW(joined_src.c_str(), joined_dst.c_str()))
|
||||
return true;
|
||||
if (GetLastError() != ERROR_ALREADY_EXISTS)
|
||||
return false;
|
||||
return bool(ReplaceFileW(joined_dst.c_str(), joined_src.c_str(), nullptr, 0, nullptr, nullptr));
|
||||
}
|
||||
|
||||
std::vector<ListEntry> OSFilesystem::list(const std::string &path)
|
||||
{
|
||||
std::vector<ListEntry> entries;
|
||||
WIN32_FIND_DATAW result;
|
||||
auto joined = Path::to_utf16(Path::join(base, path));
|
||||
joined += L"/*";
|
||||
|
||||
HANDLE handle = FindFirstFileW(joined.c_str(), &result);
|
||||
if (handle == INVALID_HANDLE_VALUE)
|
||||
return entries;
|
||||
|
||||
do
|
||||
{
|
||||
ListEntry entry;
|
||||
if (result.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
|
||||
entry.type = PathType::Directory;
|
||||
else
|
||||
entry.type = PathType::File;
|
||||
|
||||
auto utf8_path = Path::to_utf8(result.cFileName);
|
||||
if (utf8_path == "." || utf8_path == "..")
|
||||
continue;
|
||||
|
||||
entry.path = Path::join(path, utf8_path);
|
||||
entries.push_back(std::move(entry));
|
||||
} while (FindNextFileW(handle, &result));
|
||||
|
||||
FindClose(handle);
|
||||
return entries;
|
||||
}
|
||||
|
||||
bool OSFilesystem::stat(const std::string &path, FileStat &stat)
|
||||
{
|
||||
auto joined = Path::join(base, path);
|
||||
struct __stat64 buf;
|
||||
if (_wstat64(Path::to_utf16(joined).c_str(), &buf) < 0)
|
||||
return false;
|
||||
|
||||
if (buf.st_mode & _S_IFREG)
|
||||
stat.type = PathType::File;
|
||||
else if (buf.st_mode & _S_IFDIR)
|
||||
stat.type = PathType::Directory;
|
||||
else
|
||||
stat.type = PathType::Special;
|
||||
|
||||
stat.size = uint64_t(buf.st_size);
|
||||
stat.last_modified = buf.st_mtime;
|
||||
return true;
|
||||
}
|
||||
|
||||
int OSFilesystem::get_notification_fd() const
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
} // namespace Granite
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
/* Copyright (c) 2017-2026 Hans-Kristian Arntzen
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining
|
||||
* a copy of this software and associated documentation files (the
|
||||
* "Software"), to deal in the Software without restriction, including
|
||||
* without limitation the rights to use, copy, modify, merge, publish,
|
||||
* distribute, sublicense, and/or sell copies of the Software, and to
|
||||
* permit persons to whom the Software is furnished to do so, subject to
|
||||
* the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be
|
||||
* included in all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
#include "filesystem.hpp"
|
||||
#include <unordered_map>
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
|
||||
namespace Granite
|
||||
{
|
||||
class MappedFile final : public File
|
||||
{
|
||||
public:
|
||||
static FileHandle open(const std::string &path, FileMode mode);
|
||||
~MappedFile() override;
|
||||
|
||||
FileMappingHandle map_subset(uint64_t offset, size_t range) override;
|
||||
FileMappingHandle map_write(size_t size) override;
|
||||
void unmap(void *mapped, size_t range) override;
|
||||
uint64_t get_size() override;
|
||||
|
||||
private:
|
||||
bool init(const std::string &path, FileMode mode);
|
||||
HANDLE file = INVALID_HANDLE_VALUE;
|
||||
HANDLE file_mapping = nullptr;
|
||||
uint64_t size = 0;
|
||||
std::string rename_from_on_close;
|
||||
std::string rename_to_on_close;
|
||||
};
|
||||
|
||||
class OSFilesystem : public FilesystemBackend
|
||||
{
|
||||
public:
|
||||
OSFilesystem(const std::string &base);
|
||||
~OSFilesystem();
|
||||
std::vector<ListEntry> list(const std::string &path) override;
|
||||
FileHandle open(const std::string &path, FileMode mode) override;
|
||||
bool stat(const std::string &path, FileStat &stat) override;
|
||||
FileNotifyHandle install_notification(const std::string &path, std::function<void (const FileNotifyInfo &)> func) override;
|
||||
void uninstall_notification(FileNotifyHandle handle) override;
|
||||
void poll_notifications() override;
|
||||
int get_notification_fd() const override;
|
||||
std::string get_filesystem_path(const std::string &path) override;
|
||||
|
||||
bool remove(const std::string &str) override;
|
||||
bool move_yield(const std::string &dst, const std::string &src) override;
|
||||
bool move_replace(const std::string &dst, const std::string &src) override;
|
||||
|
||||
private:
|
||||
std::string base;
|
||||
|
||||
struct Handler
|
||||
{
|
||||
std::string path;
|
||||
std::function<void (const FileNotifyInfo &)> func;
|
||||
HANDLE handle = nullptr;
|
||||
HANDLE event = nullptr;
|
||||
DWORD async_buffer[1024];
|
||||
OVERLAPPED overlapped;
|
||||
};
|
||||
|
||||
std::unordered_map<FileNotifyHandle, Handler> handlers;
|
||||
FileNotifyHandle handle_id = 0;
|
||||
void kick_async(Handler &handler);
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user