Open andhuang-CLGX opened 2 years ago
The full, correct solution would probably have to be implemented by Bokeh. Somewhere around here
import panel as pn
pn.extension()
def reset(event):
file_input.disabled = False
progress.active = False
file_input = pn.widgets.FileInput()
progress = pn.widgets.Progress(active=False)
file_input.jscallback(
args={"progress": progress},
value="""
progress.active = true;
source.disabled = true;
"""
)
file_input.param.watch(reset, "value")
col = pn.Column(progress, file_input)
col.servable()
This kind of works; not immediate, but much faster to react: the jscallback tries to activate the progress bar and disable another upload, while the python watch waits until it finishes uploading to re-enable both widgets (using the bug as a feature to know when it finishes uploading :P)
So this lack of responsiveness was presenting a frustrating user experience for my labmates, so I took a crack at it. Basically everything I tried in Panel lead to the same result: Progress was updated cleanly up until the file transfer event occurs, then everything would hang for ~2 min while the actual upload took place. I really wanted to have it where I could dynamically update a progress bar as the transfer itself took place. Marc's suggestion of taking the Bokeh file_input.ts code and modifying it was extremely helpful, so thank you for that! The solution I came up with doubles as a "streaming" file input, where each file is transferred as soon as possible, so you could also potentially launch background processing tasks as data rolls in (I plan on using this for a multiprocessing queue), but the data is also saved in lists and can be handled in the normal way (This can be pretty easily removed if that functionality is not necessary). Hopefully this will serve to show how this problem might be approached in a future update. I plan on also implementing Pako to pre-compress the data before the transfer to see if that speeds things up, but that felt a bit beyond the scope of this issue. Here's the code, my apologies if there is anything non-conventional, I'm extremely new to Typescript:
_uploadtest.py
#Bokeh version: 2.4.3
#Panel version: 0.14.1
import panel as pn
from bokeh.core.properties import List, String, Bool, Int
from bokeh.layouts import column
from bokeh.models import LayoutDOM
pn.extension()
class CustomFileInputStream(LayoutDOM):
__implementation__ = "assets/ts/custom_file_inputstream.ts"
filename = String(default = "")
value = String(default = "")
mime_type = String(default = "")
accept = String(default = "")
multiple = Bool(default=False)
is_loading = Bool(default=False)
num_files = Int(default=0)
load_progress = Int(default=0)
filenames = List(String, default=[])
values = List(String, default=[])
mime_types = List(String, default=[])
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.on_change("is_loading", self._reset_lists)
self.on_change("filename", self._filename_transfered)
self.on_change("value", self._value_transfered)
self.on_change("mime_type", self._mime_type_transfered)
def _reset_lists(self, attr, old, new):
if new:
self.filenames = []
self.values = []
self.mime_types = []
def _filename_transfered(self, attr, old, new):
self.filenames.append(new)
def _value_transfered(self, attr, old, new):
self.values.append(new)
def _mime_type_transfered(self, attr, old, new):
self.mime_types.append(new)
custom_file = CustomFileInputStream(multiple = True)
def _file_loading_callback(attr, old, new):
if new:
test_text.value = f"Loading {custom_file.num_files} files...\n"
else:
test_text.value += "Loading complete!"
custom_file.on_change("is_loading", _file_loading_callback)
def _file_loading_progress(attr, old, new):
progress_bar.value = custom_file.load_progress
custom_file.on_change("load_progress", _file_loading_progress)
def _file_contents_changed(attr, old, new):
test_text.value += f"{new}\n"
custom_file.on_change("filename", _file_contents_changed)
layout = column(custom_file)
bokeh_pane = pn.pane.Bokeh(layout)
progress_bar = pn.indicators.Progress(name="ProgressBar", value=1, max=100, active=False)
test_text = pn.widgets.TextAreaInput(width=500, height=300)
check_button = pn.widgets.Button(name="Check")
def check_callback(event):
test_text.value = f"Loaded {len(custom_file.filenames)} files\n"
for f in custom_file.filenames:
test_text.value += f"{f}\n"
check_button.on_click(check_callback)
ui = pn.Column(
test_text,
progress_bar,
bokeh_pane,
check_button
)
ui.servable()
custom_file_inputstream.ts
import { input } from "core/dom"
import * as p from "core/properties"
import {Widget, WidgetView} from "models/widgets/widget"
export class CustomFileInputStreamView extends WidgetView {
override model: CustomFileInputStream
protected dialog_el: HTMLInputElement
override connect_signals(): void {
super.connect_signals()
this.connect(this.model.change, () => this.render())
}
override render(): void {
const {multiple, accept, disabled, width} = this.model
if (this.dialog_el == null) {
this.dialog_el = input({type: "file", multiple: multiple})
this.dialog_el.onchange = () => {
const {files} = this.dialog_el
if (files != null) {
this.model.setv({num_files: files.length, is_loading: true})
this.load_files(files)
}
}
this.el.appendChild(this.dialog_el)
}
if (accept != null && accept != "") {
this.dialog_el.accept = accept
}
this.dialog_el.style.width = `${width}px`
this.dialog_el.disabled = disabled
}
async load_files(files: FileList): Promise<void> {
var progress: number = 0
for (const file of files) {
const data_url = await this._read_file(file)
const [, mime_type="",, value=""] = data_url.split(/[:;,]/, 4)
progress += 1
this.model.setv({
value: value,
filename: file.name,
mime_type: mime_type,
load_progress: Math.round(100 * (progress / this.model.num_files))
})
}
this.model.setv({is_loading: false})
}
protected _read_file(file: File): Promise<string> {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => {
const {result} = reader
if (result != null) {
resolve(result as string)
} else {
reject(reader.error ?? new Error(`unable to read '${file.name}'`))
}
}
reader.readAsDataURL(file)
})
}
}
export namespace CustomFileInputStream {
export type Attrs = p.AttrsOf<Props>
export type Props = Widget.Props & {
value: p.Property<string>
mime_type: p.Property<string>
filename: p.Property<string>
accept: p.Property<string>
multiple: p.Property<boolean>
is_loading: p.Property<boolean>
num_files: p.Property<number>
load_progress: p.Property<number>
values: p.Property<string[]>
mime_types: p.Property<string[]>
filenames: p.Property<string[]>
}
}
export interface CustomFileInputStream extends CustomFileInputStream.Attrs {}
export class CustomFileInputStream extends Widget {
override properties: CustomFileInputStream.Props
override __view_type__: CustomFileInputStreamView
constructor(attrs?: Partial<CustomFileInputStream.Attrs>) {
super(attrs)
}
static {
this.prototype.default_view = CustomFileInputStreamView
this.define<CustomFileInputStream.Props>(({Number, Boolean, String, Array}) => ({
value: [ String, "" ],
mime_type: [ String, "" ],
filename: [ String, "" ],
accept: [ String, "" ],
multiple: [ Boolean, false ],
is_loading: [ Boolean, false],
num_files: [ Number, 0],
load_progress: [ Number, 0],
values: [ Array(String) ],
mime_types: [ Array(String) ],
filenames: [ Array(String) ],
}))
}
}
Hi,
Are there any updates or plans to integrate that as built-in functionality in panel or bokeh?
If not, could someone elaborate a bit more on how to implement this "patching" of bokeh in the most sustainable way? I.e., in a way that does not fail after every update.
The FileDropper
widget now allows for arbitrary chunking, so you should be able to report progress. I will add a progress
parameter to FileDropper
that automatically updates with a value between 0 and 100, so you can do something like pn.indicators.Progress(value=dropper.param.progress)
.
When uploading a big file, the watched events run very late (likely after the big file is uploaded). Is there a way to trigger
loading
before the upload so that the user knows something is running?https://panel.holoviz.org/reference/widgets/FileInput.html#widgets-gallery-fileinput