mirror of
https://github.com/thegatesbrowser/thegates.git
synced 2025-08-18 12:16:33 -04:00
parallel resource downloading
This commit is contained in:
parent
fb587a6d30
commit
fe5244d458
11 changed files with 116 additions and 40 deletions
|
@ -32,6 +32,7 @@ func is_cached(url: String) -> bool:
|
|||
|
||||
|
||||
func download(url: String, timeout: float = 0) -> String:
|
||||
if url.is_empty(): return ""
|
||||
var save_path = DOWNLOAD_FOLDER + "/" + url.md5_text() + "." + url.get_file().get_extension()
|
||||
|
||||
if has_request(save_path):
|
||||
|
@ -52,6 +53,7 @@ func download(url: String, timeout: float = 0) -> String:
|
|||
|
||||
# Returns directory where file was downloaded. Keeps filename
|
||||
func download_shared_lib(url: String, gate_url: String) -> String:
|
||||
if url.is_empty(): return ""
|
||||
var dir = DOWNLOAD_FOLDER + "/" + gate_url.md5_text()
|
||||
var save_path = dir + "/" + url.get_file()
|
||||
|
||||
|
@ -97,6 +99,7 @@ func create_request(url: String, save_path: String, timeout: float = 0) -> int:
|
|||
if err != OK: return err
|
||||
var code = (await http.request_completed)[1]
|
||||
|
||||
progress.emit(url, http.get_body_size(), http.get_downloaded_bytes())
|
||||
timer.stop()
|
||||
remove_child(timer)
|
||||
remove_child(http)
|
||||
|
|
|
@ -3,7 +3,13 @@ extends Node
|
|||
@export var gate_events: GateEvents
|
||||
@export var connect_timeout: float
|
||||
|
||||
var c_gate: ConfigGate
|
||||
var gate: Gate
|
||||
|
||||
# For parallel downloading
|
||||
var has_errors: bool
|
||||
var load_resources_done: bool
|
||||
var shared_libs_count: int = -1
|
||||
var shared_libs_done: int
|
||||
|
||||
|
||||
func _ready() -> void:
|
||||
|
@ -16,27 +22,61 @@ func load_gate(config_url: String) -> void:
|
|||
var config_path = await FileDownloader.download(config_url, connect_timeout)
|
||||
if config_path.is_empty(): return error(GateEvents.GateError.NOT_FOUND)
|
||||
|
||||
c_gate = ConfigGate.new(config_path, config_url)
|
||||
var c_gate = ConfigGate.new(config_path, config_url)
|
||||
if c_gate.load_result != OK: return error(GateEvents.GateError.INVALID_CONFIG)
|
||||
gate_events.gate_config_loaded_emit(config_url, c_gate)
|
||||
|
||||
var image_path = await FileDownloader.download(c_gate.image_url)
|
||||
var gate = Gate.create(config_url, c_gate.title, c_gate.description, image_path, "", "")
|
||||
gate = Gate.create(config_url, c_gate.title, c_gate.description, "", "", "")
|
||||
gate_events.gate_info_loaded_emit(gate)
|
||||
|
||||
# Download all in parallel
|
||||
load_image(c_gate)
|
||||
load_resources(c_gate)
|
||||
load_shared_libs(c_gate, config_url)
|
||||
|
||||
|
||||
func load_image(c_gate: ConfigGate) -> void:
|
||||
gate.image = await FileDownloader.download(c_gate.image_url)
|
||||
gate_events.gate_image_loaded_emit(gate)
|
||||
# finish without image
|
||||
|
||||
|
||||
func load_resources(c_gate: ConfigGate) -> void:
|
||||
gate.resource_pack = await FileDownloader.download(c_gate.resource_pack_url)
|
||||
if gate.resource_pack.is_empty(): return error(GateEvents.GateError.MISSING_PACK)
|
||||
|
||||
load_resources_done = true
|
||||
try_finish_loading()
|
||||
|
||||
|
||||
func load_shared_libs(c_gate: ConfigGate, config_url: String) -> void:
|
||||
Debug.logclr("GDExtension libraries: " + str(c_gate.libraries), Color.DIM_GRAY)
|
||||
shared_libs_count = c_gate.libraries.size()
|
||||
for lib in c_gate.libraries:
|
||||
gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url)
|
||||
if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS)
|
||||
load_lib(config_url, lib)
|
||||
|
||||
try_finish_loading() # In case of 0 libs
|
||||
|
||||
|
||||
func load_lib(config_url: String, lib: String) -> void:
|
||||
gate.shared_libs_dir = await FileDownloader.download_shared_lib(lib, config_url)
|
||||
if gate.shared_libs_dir.is_empty(): return error(GateEvents.GateError.MISSING_LIBS)
|
||||
|
||||
shared_libs_done += 1
|
||||
try_finish_loading()
|
||||
|
||||
|
||||
func try_finish_loading() -> void:
|
||||
if has_errors: return
|
||||
if not load_resources_done: return
|
||||
if shared_libs_count == -1 or shared_libs_done != shared_libs_count: return
|
||||
|
||||
gate_events.gate_loaded_emit(gate)
|
||||
|
||||
|
||||
func error(code: GateEvents.GateError) -> void:
|
||||
Debug.logclr("GateError: " + GateEvents.GateError.keys()[code], Color.MAROON)
|
||||
has_errors = true
|
||||
gate_events.gate_error_emit(code)
|
||||
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ signal search(query: String)
|
|||
signal open_gate(url: String)
|
||||
signal gate_config_loaded(url: String, config: ConfigGate)
|
||||
signal gate_info_loaded(gate: Gate)
|
||||
signal gate_image_loaded(gate: Gate) # might be empty image
|
||||
signal gate_loaded(gate: Gate)
|
||||
signal gate_entered
|
||||
signal first_frame
|
||||
|
@ -50,6 +51,10 @@ func gate_info_loaded_emit(gate: Gate) -> void:
|
|||
gate_info_loaded.emit(gate)
|
||||
|
||||
|
||||
func gate_image_loaded_emit(gate: Gate) -> void:
|
||||
gate_image_loaded.emit(gate)
|
||||
|
||||
|
||||
func gate_loaded_emit(gate: Gate) -> void:
|
||||
current_gate = gate
|
||||
gate_loaded.emit(gate)
|
||||
|
|
|
@ -60,17 +60,17 @@ func send_filehandle(filehandle_path: String) -> void:
|
|||
func set_texture_format(format: RenderingDevice.DataFormat) -> void:
|
||||
match format:
|
||||
RenderingDevice.DATA_FORMAT_R8G8B8A8_UNORM:
|
||||
set_param("ext_texture_is_bgra", false)
|
||||
set_param(&"ext_texture_is_bgra", false)
|
||||
Debug.logclr("External texture format is set to RGBA8", Color.DIM_GRAY)
|
||||
RenderingDevice.DATA_FORMAT_B8G8R8A8_UNORM:
|
||||
set_param("ext_texture_is_bgra", true)
|
||||
set_param(&"ext_texture_is_bgra", true)
|
||||
Debug.logclr("External texture format is set to BGRA8", Color.DIM_GRAY)
|
||||
_:
|
||||
Debug.logerr("Texture format %d is not supported" % [format])
|
||||
|
||||
|
||||
func show_render() -> void:
|
||||
set_param("show_render", true)
|
||||
set_param(&"show_render", true)
|
||||
|
||||
|
||||
func set_param(param: StringName, value: Variant) -> void:
|
||||
|
|
|
@ -16,3 +16,14 @@ static func to_alpha(text: String) -> String:
|
|||
|
||||
result = result.strip_edges()
|
||||
return result
|
||||
|
||||
|
||||
static func bytes_to_string(bytes: int) -> String:
|
||||
if bytes < 1024: return str(bytes) + "B"
|
||||
|
||||
var kb = bytes / 1024
|
||||
if kb < 1024: return str(kb) + "KB"
|
||||
|
||||
var mb = kb / 1024.0
|
||||
var text = "%.1fMB" if mb < 10.0 else "%.0fMB"
|
||||
return text % [mb]
|
||||
|
|
|
@ -18,6 +18,7 @@ func _ready() -> void:
|
|||
gate_events.search.connect(func(_query): hide_buttons())
|
||||
gate_events.exit_gate.connect(hide_buttons)
|
||||
gate_events.gate_info_loaded.connect(update_info)
|
||||
gate_events.gate_image_loaded.connect(update_info)
|
||||
|
||||
|
||||
func show_buttons(_url: String) -> void:
|
||||
|
|
|
@ -8,7 +8,7 @@ extends Control
|
|||
|
||||
|
||||
func _ready() -> void:
|
||||
gate_events.gate_info_loaded.connect(show_thumbnail)
|
||||
gate_events.gate_image_loaded.connect(show_thumbnail)
|
||||
gate_events.first_frame.connect(on_first_frame)
|
||||
ui_events.ui_mode_changed.connect(on_ui_mode_changed)
|
||||
vignette_blur.hide()
|
||||
|
@ -17,6 +17,7 @@ func _ready() -> void:
|
|||
|
||||
func show_thumbnail(gate: Gate) -> void:
|
||||
splash_screen.texture = FileTools.load_external_tex(gate.image)
|
||||
if not is_instance_valid(splash_screen.texture): return
|
||||
vignette_blur.show()
|
||||
vignette_blur.thumbnail_params()
|
||||
|
||||
|
@ -24,6 +25,7 @@ func show_thumbnail(gate: Gate) -> void:
|
|||
func on_first_frame() -> void:
|
||||
splash_screen.hide()
|
||||
click_anywhere.show()
|
||||
vignette_blur.show()
|
||||
vignette_blur.gate_started_params()
|
||||
|
||||
|
||||
|
|
|
@ -12,7 +12,7 @@ var gate: Gate
|
|||
|
||||
|
||||
func _ready() -> void:
|
||||
gate_events.gate_info_loaded.connect(display_info)
|
||||
gate_events.gate_image_loaded.connect(display_info)
|
||||
gate_events.first_frame.connect(on_first_frame)
|
||||
gate_events.gate_error.connect(on_gate_error)
|
||||
clear_info()
|
||||
|
|
|
@ -7,6 +7,10 @@ enum ProgressStatus {
|
|||
ERROR
|
||||
}
|
||||
|
||||
class DownloadItem:
|
||||
var body_size: int
|
||||
var downloaded_bytes: int
|
||||
|
||||
@export var gate_events: GateEvents
|
||||
@export var progress_bar_background: Control
|
||||
@export var progress_bar_error: Control
|
||||
|
@ -14,13 +18,16 @@ enum ProgressStatus {
|
|||
@export var label: Label
|
||||
|
||||
const TWEEN_DURATION_S = 0.2
|
||||
const SPEED_DELAY_MS = 400
|
||||
const SPEED_DELAY_MS = 300
|
||||
const ZERO_SPEED_DELAY_MS = 2000
|
||||
|
||||
var download_items: Dictionary
|
||||
|
||||
# For calculating speed
|
||||
var last_bytes: int
|
||||
var last_ticks: int
|
||||
var last_speed: String
|
||||
var last_is_zero: int
|
||||
var last_is_zero: bool
|
||||
var first_zero_ticks: int
|
||||
|
||||
|
||||
|
@ -33,25 +40,41 @@ func _ready() -> void:
|
|||
|
||||
func on_gate_info_loaded(_gate: Gate) -> void:
|
||||
gate_events.download_progress.connect(show_progress)
|
||||
last_ticks = Time.get_ticks_msec()
|
||||
|
||||
|
||||
func show_progress(_url: String, body_size: int, downloaded_bytes: int) -> void:
|
||||
func show_progress(url: String, body_size: int, downloaded_bytes: int) -> void:
|
||||
if body_size < 0: return
|
||||
|
||||
var downloaded = bytes_to_string(downloaded_bytes)
|
||||
var body = bytes_to_string(body_size)
|
||||
var speed = get_speed(downloaded_bytes)
|
||||
var item = download_items.get_or_add(url, DownloadItem.new()) as DownloadItem
|
||||
item.body_size = body_size
|
||||
item.downloaded_bytes = downloaded_bytes
|
||||
|
||||
var sum_downloaded_bytes = get_sum(&"downloaded_bytes")
|
||||
var sum_body_size = get_sum(&"body_size")
|
||||
|
||||
var downloaded = StringTools.bytes_to_string(sum_downloaded_bytes)
|
||||
var body = StringTools.bytes_to_string(sum_body_size)
|
||||
var speed = get_speed(sum_downloaded_bytes)
|
||||
|
||||
var text = "Downloading resources — %s of %s (%s/sec)" % [downloaded, body, speed]
|
||||
var progress = float(downloaded_bytes) / body_size
|
||||
var progress = float(sum_downloaded_bytes) / sum_body_size
|
||||
set_progress(text, ProgressStatus.DOWNLOADING, progress)
|
||||
|
||||
|
||||
func get_sum(property: StringName) -> int:
|
||||
var result: int = 0
|
||||
for item in download_items.values():
|
||||
result += item.get(property)
|
||||
return result
|
||||
|
||||
|
||||
func get_speed(bytes: int) -> String:
|
||||
var delta_bytes = bytes - last_bytes
|
||||
var delta_ticks = Time.get_ticks_msec() - last_ticks
|
||||
|
||||
if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty(): return last_speed
|
||||
if delta_ticks < SPEED_DELAY_MS and not last_speed.is_empty() and not last_is_zero:
|
||||
return last_speed
|
||||
|
||||
var bytes_sec = 0
|
||||
if delta_bytes != 0 and delta_ticks != 0:
|
||||
|
@ -60,13 +83,16 @@ func get_speed(bytes: int) -> String:
|
|||
if should_write_current_speed(bytes_sec):
|
||||
last_bytes = bytes
|
||||
last_ticks = Time.get_ticks_msec()
|
||||
last_speed = bytes_to_string(bytes_sec)
|
||||
last_speed = StringTools.bytes_to_string(bytes_sec)
|
||||
|
||||
return last_speed
|
||||
|
||||
|
||||
func should_write_current_speed(bytes_sec: int) -> bool:
|
||||
if last_speed.is_empty(): return true
|
||||
if last_speed.is_empty():
|
||||
last_is_zero = bytes_sec == 0
|
||||
return true
|
||||
|
||||
if bytes_sec != 0:
|
||||
last_is_zero = false
|
||||
return true
|
||||
|
@ -83,17 +109,6 @@ func should_write_current_speed(bytes_sec: int) -> bool:
|
|||
return true
|
||||
|
||||
|
||||
func bytes_to_string(bytes: int) -> String:
|
||||
if bytes < 1024: return str(bytes) + "B"
|
||||
|
||||
var kb = bytes / 1024
|
||||
if kb < 1024: return str(kb) + "KB"
|
||||
|
||||
var mb = kb / 1024.0
|
||||
var text = "%.1fMB" if mb < 10.0 else "%.0fMB"
|
||||
return text % [mb]
|
||||
|
||||
|
||||
func on_gate_entered() -> void:
|
||||
gate_events.download_progress.disconnect(show_progress)
|
||||
set_progress("Starting the gate...", ProgressStatus.STARTING)
|
||||
|
@ -118,17 +133,17 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) -
|
|||
ProgressStatus.CONNECTING:
|
||||
progress_bar.show()
|
||||
progress_bar_error.hide()
|
||||
move_progress_bar(0.0)
|
||||
move_progress_bar(0.0, 0.0)
|
||||
|
||||
ProgressStatus.DOWNLOADING:
|
||||
progress_bar.show()
|
||||
progress_bar_error.hide()
|
||||
move_progress_bar(progress)
|
||||
move_progress_bar(progress, FileDownloader.PROGRESS_DELAY)
|
||||
|
||||
ProgressStatus.STARTING:
|
||||
progress_bar.show()
|
||||
progress_bar_error.hide()
|
||||
move_progress_bar(1.0, true)
|
||||
move_progress_bar(1.0, TWEEN_DURATION_S)
|
||||
|
||||
ProgressStatus.ERROR:
|
||||
progress_bar.hide()
|
||||
|
@ -139,10 +154,9 @@ func set_progress(text: String, status: ProgressStatus, progress: float = 0.0) -
|
|||
progress_bar_error.hide()
|
||||
|
||||
|
||||
func move_progress_bar(progress: float, custom_delay: bool = false) -> void:
|
||||
func move_progress_bar(progress: float, tween_duration: float) -> void:
|
||||
var full_size = progress_bar_background.size
|
||||
var current_size = Vector2(lerp(0.0, full_size.x, progress), full_size.y)
|
||||
var tween_duration = TWEEN_DURATION_S if custom_delay else FileDownloader.PROGRESS_DELAY
|
||||
|
||||
var tween = get_tree().create_tween()
|
||||
tween.tween_property(progress_bar, "size", current_size, tween_duration)
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
extends Control
|
||||
class_name VignetteBlur
|
||||
|
||||
const BLUR_AMOUNT = "BlurAmount"
|
||||
const UV_SCALE = "UVScale"
|
||||
const BLUR_AMOUNT = &"BlurAmount"
|
||||
const UV_SCALE = &"UVScale"
|
||||
|
||||
@export var blur_amount: float
|
||||
@export var blur_amount_started: float
|
||||
|
|
|
@ -7,7 +7,7 @@ const url_regex: String = "^(https?)://[^\\s()<>]+(?:\\([\\w\\d]+\\)|([^[:punct:
|
|||
static func join(base_url: String, path: String) -> String:
|
||||
var url = ""
|
||||
if path.is_empty():
|
||||
url = base_url
|
||||
url = ""
|
||||
elif path.begins_with("http"):
|
||||
url = path
|
||||
else:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue