From fe5244d458e14990c2d6dd63ccc3f0f50cd9e8cb Mon Sep 17 00:00:00 2001 From: Nordup Date: Sat, 7 Dec 2024 13:49:51 +0400 Subject: [PATCH] parallel resource downloading --- app/scripts/loading/file_downloader.gd | 3 ++ app/scripts/loading/gate_loader.gd | 52 +++++++++++++++++--- app/scripts/resources/gate_events.gd | 5 ++ app/scripts/sandbox/render_result.gd | 6 +-- app/scripts/string_tools.gd | 11 +++++ app/scripts/ui/menu/star.gd | 1 + app/scripts/ui/world/foreground.gd | 4 +- app/scripts/ui/world/gate_info.gd | 2 +- app/scripts/ui/world/loading_status.gd | 66 ++++++++++++++++---------- app/scripts/ui/world/vignette_blur.gd | 4 +- app/scripts/url.gd | 2 +- 11 files changed, 116 insertions(+), 40 deletions(-) diff --git a/app/scripts/loading/file_downloader.gd b/app/scripts/loading/file_downloader.gd index 733091a..97c2862 100644 --- a/app/scripts/loading/file_downloader.gd +++ b/app/scripts/loading/file_downloader.gd @@ -32,6 +32,7 @@ func is_cached(url: String) -> bool: func download(url: String, timeout: float = 0) -> String: + if url.is_empty(): return "" var save_path = DOWNLOAD_FOLDER + "/" + url.md5_text() + "." + url.get_file().get_extension() if has_request(save_path): @@ -52,6 +53,7 @@ func download(url: String, timeout: float = 0) -> String: # Returns directory where file was downloaded. Keeps filename func download_shared_lib(url: String, gate_url: String) -> String: + if url.is_empty(): return "" var dir = DOWNLOAD_FOLDER + "/" + gate_url.md5_text() var save_path = dir + "/" + url.get_file() @@ -97,6 +99,7 @@ func create_request(url: String, save_path: String, timeout: float = 0) -> int: if err != OK: return err var code = (await http.request_completed)[1] + progress.emit(url, http.get_body_size(), http.get_downloaded_bytes()) timer.stop() remove_child(timer) remove_child(http) diff --git a/app/scripts/loading/gate_loader.gd b/app/scripts/loading/gate_loader.gd index 1145cbe..dbfb4a7 100644 --- a/app/scripts/loading/gate_loader.gd +++ b/app/scripts/loading/gate_loader.gd @@ -3,7 +3,13 @@ extends Node @export var gate_events: GateEvents @export var connect_timeout: float -var c_gate: ConfigGate +var gate: Gate + +# For parallel downloading +var has_errors: bool +var load_resources_done: bool +var shared_libs_count: int = -1 +var shared_libs_done: int func _ready() -> void: @@ -16,27 +22,61 @@ func load_gate(config_url: String) -> void: var config_path = await FileDownloader.download(config_url, connect_timeout) if config_path.is_empty(): return error(GateEvents.GateError.NOT_FOUND) - c_gate = ConfigGate.new(config_path, config_url) + var c_gate = ConfigGate.new(config_path, config_url) if c_gate.load_result != OK: return error(GateEvents.GateError.INVALID_CONFIG) gate_events.gate_config_loaded_emit(config_url, c_gate) - var image_path = await FileDownloader.download(c_gate.image_url) - var gate = Gate.create(config_url, c_gate.title, c_gate.description, image_path, "", "") + gate = Gate.create(config_url, c_gate.title, c_gate.description, "", "", "") gate_events.gate_info_loaded_emit(gate) + # Download all in parallel + load_image(c_gate) + load_resources(c_gate) + load_shared_libs(c_gate, config_url) + + +func load_image(c_gate: ConfigGate) -> void: + gate.image = await FileDownloader.download(c_gate.image_url) + gate_events.gate_image_loaded_emit(gate) + # finish without image + + +func load_resources(c_gate: ConfigGate) -> void: gate.resource_pack = await FileDownloader.download(c_gate.resource_pack_url) if gate.resource_pack.is_empty(): return error(GateEvents.GateError.MISSING_PACK) + load_resources_done = true + try_finish_loading() + + +func load_shared_libs(c_gate: ConfigGate, config_url: String) -> void: Debug.logclr("GDExtension libraries: " + str(c_gate.libraries), Color.DIM_GRAY) + shared_libs_count = c_gate.libraries.size() for lib in c_gate.libraries: - gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url) - if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS) + load_lib(config_url, lib) + + try_finish_loading() # In case of 0 libs + + +func load_lib(config_url: String, lib: String) -> void: + gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url) + if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS) + + shared_libs_done += 1 + try_finish_loading() + + +func try_finish_loading() -> void: + if has_errors: return + if not load_resources_done: return + if shared_libs_count == -1 or shared_libs_done != shared_libs_count: return gate_events.gate_loaded_emit(gate) func error(code: GateEvents.GateError) -> void: Debug.logclr("GateError: " + GateEvents.GateError.keys()[code], Color.MAROON) + has_errors = true gate_events.gate_error_emit(code) diff --git a/app/scripts/resources/gate_events.gd b/app/scripts/resources/gate_events.gd index 1463a97..fb34586 100644 --- a/app/scripts/resources/gate_events.gd +++ b/app/scripts/resources/gate_events.gd @@ -5,6 +5,7 @@ signal search(query: String) signal open_gate(url: String) signal gate_config_loaded(url: String, config: ConfigGate) signal gate_info_loaded(gate: Gate) +signal gate_image_loaded(gate: Gate) # might be empty image signal gate_loaded(gate: Gate) signal gate_entered signal first_frame @@ -50,6 +51,10 @@ func gate_info_loaded_emit(gate: Gate) -> void: gate_info_loaded.emit(gate) +func gate_image_loaded_emit(gate: Gate) -> void: + gate_image_loaded.emit(gate) + + func gate_loaded_emit(gate: Gate) -> void: current_gate = gate gate_loaded.emit(gate) diff --git a/app/scripts/sandbox/render_result.gd b/app/scripts/sandbox/render_result.gd index 1a761f8..6a0ff35 100644 --- a/app/scripts/sandbox/render_result.gd +++ b/app/scripts/sandbox/render_result.gd @@ -60,17 +60,17 @@ func send_filehandle(filehandle_path: String) -> void: func set_texture_format(format: RenderingDevice.DataFormat) -> void: match format: RenderingDevice.DATA_FORMAT_R8G8B8A8_UNORM: - set_param("ext_texture_is_bgra", false) + set_param(&"ext_texture_is_bgra", false) Debug.logclr("External texture format is set to RGBA8", Color.DIM_GRAY) RenderingDevice.DATA_FORMAT_B8G8R8A8_UNORM: - set_param("ext_texture_is_bgra", true) + set_param(&"ext_texture_is_bgra", true) Debug.logclr("External texture format is set to BGRA8", Color.DIM_GRAY) _: Debug.logerr("Texture format %d is not supported" % [format]) func show_render() -> void: - set_param("show_render", true) + set_param(&"show_render", true) func set_param(param: StringName, value: Variant) -> void: diff --git a/app/scripts/string_tools.gd b/app/scripts/string_tools.gd index 1b4d742..7e7ff3e 100644 --- a/app/scripts/string_tools.gd +++ b/app/scripts/string_tools.gd @@ -16,3 +16,14 @@ static func to_alpha(text: String) -> String: result = result.strip_edges() return result + + +static func bytes_to_string(bytes: int) -> String: + if bytes < 1024: return str(bytes) + "B" + + var kb = bytes / 1024 + if kb < 1024: return str(kb) + "KB" + + var mb = kb / 1024.0 + var text = "%.1fMB" if mb < 10.0 else "%.0fMB" + return text % [mb] diff --git a/app/scripts/ui/menu/star.gd b/app/scripts/ui/menu/star.gd index 1b8de5c..ce88e9f 100644 --- a/app/scripts/ui/menu/star.gd +++ b/app/scripts/ui/menu/star.gd @@ -18,6 +18,7 @@ func _ready() -> void: gate_events.search.connect(func(_query): hide_buttons()) gate_events.exit_gate.connect(hide_buttons) gate_events.gate_info_loaded.connect(update_info) + gate_events.gate_image_loaded.connect(update_info) func show_buttons(_url: String) -> void: diff --git a/app/scripts/ui/world/foreground.gd b/app/scripts/ui/world/foreground.gd index 57f46f3..2fea619 100644 --- a/app/scripts/ui/world/foreground.gd +++ b/app/scripts/ui/world/foreground.gd @@ -8,7 +8,7 @@ extends Control func _ready() -> void: - gate_events.gate_info_loaded.connect(show_thumbnail) + gate_events.gate_image_loaded.connect(show_thumbnail) gate_events.first_frame.connect(on_first_frame) ui_events.ui_mode_changed.connect(on_ui_mode_changed) vignette_blur.hide() @@ -17,6 +17,7 @@ func _ready() -> void: func show_thumbnail(gate: Gate) -> void: splash_screen.texture = FileTools.load_external_tex(gate.image) + if not is_instance_valid(splash_screen.texture): return vignette_blur.show() vignette_blur.thumbnail_params() @@ -24,6 +25,7 @@ func show_thumbnail(gate: Gate) -> void: func on_first_frame() -> void: splash_screen.hide() click_anywhere.show() + vignette_blur.show() vignette_blur.gate_started_params() diff --git a/app/scripts/ui/world/gate_info.gd b/app/scripts/ui/world/gate_info.gd index a5b4f8d..d2848a6 100644 --- a/app/scripts/ui/world/gate_info.gd +++ b/app/scripts/ui/world/gate_info.gd @@ -12,7 +12,7 @@ var gate: Gate func _ready() -> void: - gate_events.gate_info_loaded.connect(display_info) + gate_events.gate_image_loaded.connect(display_info) gate_events.first_frame.connect(on_first_frame) gate_events.gate_error.connect(on_gate_error) clear_info() diff --git a/app/scripts/ui/world/loading_status.gd b/app/scripts/ui/world/loading_status.gd index c36c23e..be430ad 100644 --- a/app/scripts/ui/world/loading_status.gd +++ b/app/scripts/ui/world/loading_status.gd @@ -7,6 +7,10 @@ enum ProgressStatus { ERROR } +class DownloadItem: + var body_size: int + var downloaded_bytes: int + @export var gate_events: GateEvents @export var progress_bar_background: Control @export var progress_bar_error: Control @@ -14,13 +18,16 @@ enum ProgressStatus { @export var label: Label const TWEEN_DURATION_S = 0.2 -const SPEED_DELAY_MS = 400 +const SPEED_DELAY_MS = 300 const ZERO_SPEED_DELAY_MS = 2000 +var download_items: Dictionary + +# For calculating speed var last_bytes: int var last_ticks: int var last_speed: String -var last_is_zero: int +var last_is_zero: bool var first_zero_ticks: int @@ -33,25 +40,41 @@ func _ready() -> void: func on_gate_info_loaded(_gate: Gate) -> void: gate_events.download_progress.connect(show_progress) + last_ticks = Time.get_ticks_msec() -func show_progress(_url: String, body_size: int, downloaded_bytes: int) -> void: +func show_progress(url: String, body_size: int, downloaded_bytes: int) -> void: if body_size < 0: return - var downloaded = bytes_to_string(downloaded_bytes) - var body = bytes_to_string(body_size) - var speed = get_speed(downloaded_bytes) + var item = download_items.get_or_add(url, DownloadItem.new()) as DownloadItem + item.body_size = body_size + item.downloaded_bytes = downloaded_bytes + + var sum_downloaded_bytes = get_sum(&"downloaded_bytes") + var sum_body_size = get_sum(&"body_size") + + var downloaded = StringTools.bytes_to_string(sum_downloaded_bytes) + var body = StringTools.bytes_to_string(sum_body_size) + var speed = get_speed(sum_downloaded_bytes) var text = "Downloading resources — %s of %s (%s/sec)" % [downloaded, body, speed] - var progress = float(downloaded_bytes) / body_size + var progress = float(sum_downloaded_bytes) / sum_body_size set_progress(text, ProgressStatus.DOWNLOADING, progress) +func get_sum(property: StringName) -> int: + var result: int = 0 + for item in download_items.values(): + result += item.get(property) + return result + + func get_speed(bytes: int) -> String: var delta_bytes = bytes - last_bytes var delta_ticks = Time.get_ticks_msec() - last_ticks - if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty(): return last_speed + if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty() and not last_is_zero: + return last_speed var bytes_sec = 0 if delta_bytes != 0 and delta_ticks != 0: @@ -60,13 +83,16 @@ func get_speed(bytes: int) -> String: if should_write_current_speed(bytes_sec): last_bytes = bytes last_ticks = Time.get_ticks_msec() - last_speed = bytes_to_string(bytes_sec) + last_speed = StringTools.bytes_to_string(bytes_sec) return last_speed func should_write_current_speed(bytes_sec: int) -> bool: - if last_speed.is_empty(): return true + if last_speed.is_empty(): + last_is_zero = bytes_sec == 0 + return true + if bytes_sec != 0: last_is_zero = false return true @@ -83,17 +109,6 @@ func should_write_current_speed(bytes_sec: int) -> bool: return true -func bytes_to_string(bytes: int) -> String: - if bytes < 1024: return str(bytes) + "B" - - var kb = bytes / 1024 - if kb < 1024: return str(kb) + "KB" - - var mb = kb / 1024.0 - var text = "%.1fMB" if mb < 10.0 else "%.0fMB" - return text % [mb] - - func on_gate_entered() -> void: gate_events.download_progress.disconnect(show_progress) set_progress("Starting the gate...", ProgressStatus.STARTING) @@ -118,17 +133,17 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) - ProgressStatus.CONNECTING: progress_bar.show() progress_bar_error.hide() - move_progress_bar(0.0) + move_progress_bar(0.0, 0.0) ProgressStatus.DOWNLOADING: progress_bar.show() progress_bar_error.hide() - move_progress_bar(progress) + move_progress_bar(progress, FileDownloader.PROGRESS_DELAY) ProgressStatus.STARTING: progress_bar.show() progress_bar_error.hide() - move_progress_bar(1.0, true) + move_progress_bar(1.0, TWEEN_DURATION_S) ProgressStatus.ERROR: progress_bar.hide() @@ -139,10 +154,9 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) - progress_bar_error.hide() -func move_progress_bar(progress: float, custom_delay: bool = false) -> void: +func move_progress_bar(progress: float, tween_duration: float) -> void: var full_size = progress_bar_background.size var current_size = Vector2(lerp(0.0, full_size.x, progress), full_size.y) - var tween_duration = TWEEN_DURATION_S if custom_delay else FileDownloader.PROGRESS_DELAY var tween = get_tree().create_tween() tween.tween_property(progress_bar, "size", current_size, tween_duration) diff --git a/app/scripts/ui/world/vignette_blur.gd b/app/scripts/ui/world/vignette_blur.gd index ddd40eb..5c9a108 100644 --- a/app/scripts/ui/world/vignette_blur.gd +++ b/app/scripts/ui/world/vignette_blur.gd @@ -1,8 +1,8 @@ extends Control class_name VignetteBlur -const BLUR_AMOUNT = "BlurAmount" -const UV_SCALE = "UVScale" +const BLUR_AMOUNT = &"BlurAmount" +const UV_SCALE = &"UVScale" @export var blur_amount: float @export var blur_amount_started: float diff --git a/app/scripts/url.gd b/app/scripts/url.gd index b41ac6a..f72f424 100644 --- a/app/scripts/url.gd +++ b/app/scripts/url.gd @@ -7,7 +7,7 @@ const url_regex: String = "^(https?)://[^\\s()<>]+(?:\\([\\w\\d]+\\)|([^[:punct: static func join(base_url: String, path: String) -> String: var url = "" if path.is_empty(): - url = base_url + url = "" elif path.begins_with("http"): url = path else: