parallel resource downloading

This commit is contained in:
Nordup 2024-12-07 13:49:51 +04:00
parent fb587a6d30
commit fe5244d458
11 changed files with 116 additions and 40 deletions

View file

@ -32,6 +32,7 @@ func is_cached(url: String) -> bool:
func download(url: String, timeout: float = 0) -> String:
if url.is_empty(): return ""
var save_path = DOWNLOAD_FOLDER + "/" + url.md5_text() + "." + url.get_file().get_extension()
if has_request(save_path):
@ -52,6 +53,7 @@ func download(url: String, timeout: float = 0) -> String:
# Returns directory where file was downloaded. Keeps filename
func download_shared_lib(url: String, gate_url: String) -> String:
if url.is_empty(): return ""
var dir = DOWNLOAD_FOLDER + "/" + gate_url.md5_text()
var save_path = dir + "/" + url.get_file()
@ -97,6 +99,7 @@ func create_request(url: String, save_path: String, timeout: float = 0) -> int:
if err != OK: return err
var code = (await http.request_completed)[1]
progress.emit(url, http.get_body_size(), http.get_downloaded_bytes())
timer.stop()
remove_child(timer)
remove_child(http)

View file

@ -3,7 +3,13 @@ extends Node
@export var gate_events: GateEvents
@export var connect_timeout: float
var c_gate: ConfigGate
var gate: Gate
# For parallel downloading
var has_errors: bool
var load_resources_done: bool
var shared_libs_count: int = -1
var shared_libs_done: int
func _ready() -> void:
@ -16,27 +22,61 @@ func load_gate(config_url: String) -> void:
var config_path = await FileDownloader.download(config_url, connect_timeout)
if config_path.is_empty(): return error(GateEvents.GateError.NOT_FOUND)
c_gate = ConfigGate.new(config_path, config_url)
var c_gate = ConfigGate.new(config_path, config_url)
if c_gate.load_result != OK: return error(GateEvents.GateError.INVALID_CONFIG)
gate_events.gate_config_loaded_emit(config_url, c_gate)
var image_path = await FileDownloader.download(c_gate.image_url)
var gate = Gate.create(config_url, c_gate.title, c_gate.description, image_path, "", "")
gate = Gate.create(config_url, c_gate.title, c_gate.description, "", "", "")
gate_events.gate_info_loaded_emit(gate)
# Download all in parallel
load_image(c_gate)
load_resources(c_gate)
load_shared_libs(c_gate, config_url)
func load_image(c_gate: ConfigGate) -> void:
gate.image = await FileDownloader.download(c_gate.image_url)
gate_events.gate_image_loaded_emit(gate)
# finish without image
func load_resources(c_gate: ConfigGate) -> void:
gate.resource_pack = await FileDownloader.download(c_gate.resource_pack_url)
if gate.resource_pack.is_empty(): return error(GateEvents.GateError.MISSING_PACK)
load_resources_done = true
try_finish_loading()
func load_shared_libs(c_gate: ConfigGate, config_url: String) -> void:
Debug.logclr("GDExtension libraries: " + str(c_gate.libraries), Color.DIM_GRAY)
shared_libs_count = c_gate.libraries.size()
for lib in c_gate.libraries:
gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url)
if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS)
load_lib(config_url, lib)
try_finish_loading() # In case of 0 libs
func load_lib(config_url: String, lib: String) -> void:
gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url)
if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS)
shared_libs_done += 1
try_finish_loading()
func try_finish_loading() -> void:
if has_errors: return
if not load_resources_done: return
if shared_libs_count == -1 or shared_libs_done != shared_libs_count: return
gate_events.gate_loaded_emit(gate)
func error(code: GateEvents.GateError) -> void:
Debug.logclr("GateError: " + GateEvents.GateError.keys()[code], Color.MAROON)
has_errors = true
gate_events.gate_error_emit(code)

View file

@ -5,6 +5,7 @@ signal search(query: String)
signal open_gate(url: String)
signal gate_config_loaded(url: String, config: ConfigGate)
signal gate_info_loaded(gate: Gate)
signal gate_image_loaded(gate: Gate) # might be empty image
signal gate_loaded(gate: Gate)
signal gate_entered
signal first_frame
@ -50,6 +51,10 @@ func gate_info_loaded_emit(gate: Gate) -> void:
gate_info_loaded.emit(gate)
func gate_image_loaded_emit(gate: Gate) -> void:
gate_image_loaded.emit(gate)
func gate_loaded_emit(gate: Gate) -> void:
current_gate = gate
gate_loaded.emit(gate)

View file

@ -60,17 +60,17 @@ func send_filehandle(filehandle_path: String) -> void:
func set_texture_format(format: RenderingDevice.DataFormat) -> void:
match format:
RenderingDevice.DATA_FORMAT_R8G8B8A8_UNORM:
set_param("ext_texture_is_bgra", false)
set_param(&"ext_texture_is_bgra", false)
Debug.logclr("External texture format is set to RGBA8", Color.DIM_GRAY)
RenderingDevice.DATA_FORMAT_B8G8R8A8_UNORM:
set_param("ext_texture_is_bgra", true)
set_param(&"ext_texture_is_bgra", true)
Debug.logclr("External texture format is set to BGRA8", Color.DIM_GRAY)
_:
Debug.logerr("Texture format %d is not supported" % [format])
func show_render() -> void:
set_param("show_render", true)
set_param(&"show_render", true)
func set_param(param: StringName, value: Variant) -> void:

View file

@ -16,3 +16,14 @@ static func to_alpha(text: String) -> String:
result = result.strip_edges()
return result
static func bytes_to_string(bytes: int) -> String:
if bytes < 1024: return str(bytes) + "B"
var kb = bytes / 1024
if kb < 1024: return str(kb) + "KB"
var mb = kb / 1024.0
var text = "%.1fMB" if mb < 10.0 else "%.0fMB"
return text % [mb]

View file

@ -18,6 +18,7 @@ func _ready() -> void:
gate_events.search.connect(func(_query): hide_buttons())
gate_events.exit_gate.connect(hide_buttons)
gate_events.gate_info_loaded.connect(update_info)
gate_events.gate_image_loaded.connect(update_info)
func show_buttons(_url: String) -> void:

View file

@ -8,7 +8,7 @@ extends Control
func _ready() -> void:
gate_events.gate_info_loaded.connect(show_thumbnail)
gate_events.gate_image_loaded.connect(show_thumbnail)
gate_events.first_frame.connect(on_first_frame)
ui_events.ui_mode_changed.connect(on_ui_mode_changed)
vignette_blur.hide()
@ -17,6 +17,7 @@ func _ready() -> void:
func show_thumbnail(gate: Gate) -> void:
splash_screen.texture = FileTools.load_external_tex(gate.image)
if not is_instance_valid(splash_screen.texture): return
vignette_blur.show()
vignette_blur.thumbnail_params()
@ -24,6 +25,7 @@ func show_thumbnail(gate: Gate) -> void:
func on_first_frame() -> void:
splash_screen.hide()
click_anywhere.show()
vignette_blur.show()
vignette_blur.gate_started_params()

View file

@ -12,7 +12,7 @@ var gate: Gate
func _ready() -> void:
gate_events.gate_info_loaded.connect(display_info)
gate_events.gate_image_loaded.connect(display_info)
gate_events.first_frame.connect(on_first_frame)
gate_events.gate_error.connect(on_gate_error)
clear_info()

View file

@ -7,6 +7,10 @@ enum ProgressStatus {
ERROR
}
class DownloadItem:
var body_size: int
var downloaded_bytes: int
@export var gate_events: GateEvents
@export var progress_bar_background: Control
@export var progress_bar_error: Control
@ -14,13 +18,16 @@ enum ProgressStatus {
@export var label: Label
const TWEEN_DURATION_S = 0.2
const SPEED_DELAY_MS = 400
const SPEED_DELAY_MS = 300
const ZERO_SPEED_DELAY_MS = 2000
var download_items: Dictionary
# For calculating speed
var last_bytes: int
var last_ticks: int
var last_speed: String
var last_is_zero: int
var last_is_zero: bool
var first_zero_ticks: int
@ -33,25 +40,41 @@ func _ready() -> void:
func on_gate_info_loaded(_gate: Gate) -> void:
gate_events.download_progress.connect(show_progress)
last_ticks = Time.get_ticks_msec()
func show_progress(_url: String, body_size: int, downloaded_bytes: int) -> void:
func show_progress(url: String, body_size: int, downloaded_bytes: int) -> void:
if body_size < 0: return
var downloaded = bytes_to_string(downloaded_bytes)
var body = bytes_to_string(body_size)
var speed = get_speed(downloaded_bytes)
var item = download_items.get_or_add(url, DownloadItem.new()) as DownloadItem
item.body_size = body_size
item.downloaded_bytes = downloaded_bytes
var sum_downloaded_bytes = get_sum(&"downloaded_bytes")
var sum_body_size = get_sum(&"body_size")
var downloaded = StringTools.bytes_to_string(sum_downloaded_bytes)
var body = StringTools.bytes_to_string(sum_body_size)
var speed = get_speed(sum_downloaded_bytes)
var text = "Downloading resources — %s of %s (%s/sec)" % [downloaded, body, speed]
var progress = float(downloaded_bytes) / body_size
var progress = float(sum_downloaded_bytes) / sum_body_size
set_progress(text, ProgressStatus.DOWNLOADING, progress)
func get_sum(property: StringName) -> int:
var result: int = 0
for item in download_items.values():
result += item.get(property)
return result
func get_speed(bytes: int) -> String:
var delta_bytes = bytes - last_bytes
var delta_ticks = Time.get_ticks_msec() - last_ticks
if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty(): return last_speed
if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty() and not last_is_zero:
return last_speed
var bytes_sec = 0
if delta_bytes != 0 and delta_ticks != 0:
@ -60,13 +83,16 @@ func get_speed(bytes: int) -> String:
if should_write_current_speed(bytes_sec):
last_bytes = bytes
last_ticks = Time.get_ticks_msec()
last_speed = bytes_to_string(bytes_sec)
last_speed = StringTools.bytes_to_string(bytes_sec)
return last_speed
func should_write_current_speed(bytes_sec: int) -> bool:
if last_speed.is_empty(): return true
if last_speed.is_empty():
last_is_zero = bytes_sec == 0
return true
if bytes_sec != 0:
last_is_zero = false
return true
@ -83,17 +109,6 @@ func should_write_current_speed(bytes_sec: int) -> bool:
return true
func bytes_to_string(bytes: int) -> String:
if bytes < 1024: return str(bytes) + "B"
var kb = bytes / 1024
if kb < 1024: return str(kb) + "KB"
var mb = kb / 1024.0
var text = "%.1fMB" if mb < 10.0 else "%.0fMB"
return text % [mb]
func on_gate_entered() -> void:
gate_events.download_progress.disconnect(show_progress)
set_progress("Starting the gate...", ProgressStatus.STARTING)
@ -118,17 +133,17 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) -
ProgressStatus.CONNECTING:
progress_bar.show()
progress_bar_error.hide()
move_progress_bar(0.0)
move_progress_bar(0.0, 0.0)
ProgressStatus.DOWNLOADING:
progress_bar.show()
progress_bar_error.hide()
move_progress_bar(progress)
move_progress_bar(progress, FileDownloader.PROGRESS_DELAY)
ProgressStatus.STARTING:
progress_bar.show()
progress_bar_error.hide()
move_progress_bar(1.0, true)
move_progress_bar(1.0, TWEEN_DURATION_S)
ProgressStatus.ERROR:
progress_bar.hide()
@ -139,10 +154,9 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) -
progress_bar_error.hide()
func move_progress_bar(progress: float, custom_delay: bool = false) -> void:
func move_progress_bar(progress: float, tween_duration: float) -> void:
var full_size = progress_bar_background.size
var current_size = Vector2(lerp(0.0, full_size.x, progress), full_size.y)
var tween_duration = TWEEN_DURATION_S if custom_delay else FileDownloader.PROGRESS_DELAY
var tween = get_tree().create_tween()
tween.tween_property(progress_bar, "size", current_size, tween_duration)

View file

@ -1,8 +1,8 @@
extends Control
class_name VignetteBlur
const BLUR_AMOUNT = "BlurAmount"
const UV_SCALE = "UVScale"
const BLUR_AMOUNT = &"BlurAmount"
const UV_SCALE = &"UVScale"
@export var blur_amount: float
@export var blur_amount_started: float

View file

@ -7,7 +7,7 @@ const url_regex: String = "^(https?)://[^\\s()<>]+(?:\\([\\w\\d]+\\)|([^[:punct:
static func join(base_url: String, path: String) -> String:
var url = ""
if path.is_empty():
url = base_url
url = ""
elif path.begins_with("http"):
url = path
else: