Skip to content

Commit

Permalink
增加快速文件互传功能（基于 MSD）
Browse files Browse the repository at this point in the history 
为 MSD 路径添加配置选项
为 文件镜像名称添加配置选项
修复 make 测试环境
  • Loading branch information
@mofeng-git
mofeng-git committed Nov 22, 2024
1 parent 5450d72 commit b8ddf7c
Show file tree
Hide file tree
Showing 14 changed files with 262 additions and 36 deletions.
8 changes: 4 additions & 4 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ tox: testenv
&& cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \
&& cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \
&& mkdir -p /etc/kvmd/override.d \
&& cp /src/testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \
&& cd /src \
Expand Down Expand Up @@ -155,7 +155,7 @@ run-cfg: testenv
&& cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \
&& cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \
&& mkdir -p /etc/kvmd/override.d \
&& cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \
&& $(if $(CMD),$(CMD),python -m kvmd.apps.kvmd -m) \
Expand All @@ -178,7 +178,7 @@ run-ipmi: testenv
&& cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \
&& cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \
&& mkdir -p /etc/kvmd/override.d \
&& cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \
&& $(if $(CMD),$(CMD),python -m kvmd.apps.ipmi --run) \
Expand All @@ -201,7 +201,7 @@ run-vnc: testenv
&& cp /usr/share/kvmd/configs.default/kvmd/*.yaml /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*passwd /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/*.secret /etc/kvmd \
&& cp /usr/share/kvmd/configs.default/kvmd/main/$(if $(P),$(P),$(DEFAULT_PLATFORM)).yaml /etc/kvmd/main.yaml \
&& cp /usr/share/kvmd/configs.default/kvmd/main.yaml /etc/kvmd/main.yaml \
&& mkdir -p /etc/kvmd/override.d \
&& cp /testenv/$(if $(P),$(P),$(DEFAULT_PLATFORM)).override.yaml /etc/kvmd/override.yaml \
&& $(if $(CMD),$(CMD),python -m kvmd.apps.vnc --run) \
Expand Down
3 changes: 2 additions & 1 deletion build/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ ENV TZ=Asia/Shanghai

RUN cp /tmp/lib/* /lib/*-linux-*/ \
&& pip install --no-cache-dir --root-user-action=ignore --disable-pip-version-check /tmp/wheel/*.whl \
&& pip install --no-cache-dir --root-user-action=ignore --disable-pip-version-check pyfatfs \
&& rm -rf /tmp/lib /tmp/wheel

RUN sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/' /etc/apt/sources.list.d/debian.sources \
Expand All @@ -31,7 +32,7 @@ RUN if [ ${TARGETARCH} = arm ]; then ARCH=armhf; elif [ ${TARGETARCH} = arm64 ];
&& chmod +x /usr/local/bin/ttyd \
&& adduser kvmd --gecos "" --disabled-password \
&& ln -sf /usr/share/tesseract-ocr/*/tessdata /usr/share/tessdata \
&& mkdir -p /etc/kvmd_backup/override.d /var/lib/kvmd/msd/images /var/lib/kvmd/msd/meta /var/lib/kvmd/pst/data /opt/vc/bin /run/kvmd /tmp/kvmd-nginx \
&& mkdir -p /etc/kvmd_backup/override.d /var/lib/kvmd/msd/images /var/lib/kvmd/msd/meta /var/lib/kvmd/pst/data /var/lib/kvmd/msd/NormalFiles /opt/vc/bin /run/kvmd /tmp/kvmd-nginx \
&& touch /run/kvmd/ustreamer.sock


Expand Down
3 changes: 3 additions & 0 deletions configs/kvmd/override.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ kvmd:
msd:
#type: otg
remount_cmd: /bin/true
msd_path: /var/lib/kvmd/msd
normalfiles_path: NormalFiles
normalfiles_size: 256

ocr:
langs:
Expand Down
5 changes: 5 additions & 0 deletions kvmd/apps/kvmd/api/msd.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ async def __set_params_handler(self, req: Request) -> Response:
async def __set_connected_handler(self, req: Request) -> Response:
await self.__msd.set_connected(valid_bool(req.query.get("connected")))
return make_json_response()

@exposed_http("POST", "/msd/make_image")
async def __set_zipped_handler(self, req: Request) -> Response:
await self.__msd.make_image(valid_bool(req.query.get("zipped")))
return make_json_response()

# =====

Expand Down
10 changes: 5 additions & 5 deletions kvmd/fstab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,21 +37,21 @@ class Partition:


# =====
def find_msd() -> Partition:
return _find_single("otgmsd")
def find_msd(msd_directory_path) -> Partition:
return _find_single("otgmsd", msd_directory_path)


def find_pst() -> Partition:
return _find_single("pst")


# =====
def _find_single(part_type: str) -> Partition:
def _find_single(part_type: str, msd_directory_path: str) -> Partition:
parts = _find_partitions(part_type, True)
if len(parts) == 0:
if os.path.exists('/var/lib/kvmd/msd'):
if os.path.exists(msd_directory_path):
#set default value
parts = [Partition(mount_path='/var/lib/kvmd/msd', root_path='/var/lib/kvmd/msd',group='kvmd', user='kvmd')]
parts = [Partition(mount_path = msd_directory_path, root_path = msd_directory_path, group = 'kvmd', user = 'kvmd')]
else:
raise RuntimeError(f"Can't find {part_type!r} mountpoint")
return parts[0]
Expand Down
3 changes: 3 additions & 0 deletions kvmd/plugins/msd/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ async def set_params(

async def set_connected(self, connected: bool) -> None:
raise NotImplementedError()

async def make_image(self, zipped: bool) -> None:
raise NotImplementedError()

@contextlib.asynccontextmanager
async def read_image(self, name: str) -> AsyncGenerator[BaseMsdReader, None]:
Expand Down
122 changes: 111 additions & 11 deletions kvmd/plugins/msd/otg/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@
import time
import os
import copy
import pyfatfs
import pyfatfs.PyFat
import pyfatfs.PyFatFS

from typing import AsyncGenerator

Expand All @@ -37,8 +40,8 @@
from ....yamlconf import Option

from ....validators.basic import valid_bool
from ....validators.basic import valid_number
from ....validators.os import valid_command
from ....validators.basic import valid_number, valid_stripped_string_not_empty
from ....validators.os import valid_command, valid_abs_path
from ....validators.kvm import valid_msd_image_name

from .... import aiotools
Expand Down Expand Up @@ -120,23 +123,30 @@ def __init__( # pylint: disable=super-init-not-called
read_chunk_size: int,
write_chunk_size: int,
sync_chunk_size: int,
normalfiles_size: int,

remount_cmd: list[str],

initial: dict,

normalfiles_path: str,
msd_path: str,

gadget: str, # XXX: Not from options, see /kvmd/apps/kvmd/__init__.py for details
) -> None:

self.__read_chunk_size = read_chunk_size
self.__write_chunk_size = write_chunk_size
self.__sync_chunk_size = sync_chunk_size
self.__normalfiles_path = normalfiles_path
self.__msd_path = msd_path
self.__normalfiles_size = normalfiles_size

self.__initial_image: str = initial["image"]
self.__initial_cdrom: bool = initial["cdrom"]

self.__drive = Drive(gadget, instance=0, lun=0)
self.__storage = Storage(fstab.find_msd().root_path, remount_cmd)
self.__storage = Storage(fstab.find_msd(msd_path).root_path, remount_cmd)

self.__reader: (MsdFileReader | None) = None
self.__writer: (MsdFileWriter | None) = None
Expand Down Expand Up @@ -165,10 +175,15 @@ def get_plugin_options(cls) -> dict:
"image": Option("", type=valid_msd_image_name, if_empty=""),
"cdrom": Option(False, type=valid_bool),
},
"msd_path": Option("/var/lib/kvmd/msd", type=valid_abs_path),
"normalfiles_path": Option("NormalFiles", type=valid_stripped_string_not_empty),
"normalfiles_size": Option(256, type=functools.partial(valid_number, min=64)),
}

# =====

# =====

async def get_state(self) -> dict:
async with self.__state._lock: # pylint: disable=protected-access
storage: (dict | None) = None
Expand All @@ -184,6 +199,7 @@ async def get_state(self) -> dict:

storage["downloading"] = (self.__reader.get_state() if self.__reader else None)
storage["uploading"] = (self.__writer.get_state() if self.__writer else None)
storage["filespath"] = self.__normalfiles_path

vd: (dict | None) = None
if self.__state.vd:
Expand Down Expand Up @@ -286,15 +302,19 @@ async def set_connected(self, connected: bool) -> None:

self.__drive.set_rw_flag(self.__state.vd.rw)
self.__drive.set_cdrom_flag(self.__state.vd.cdrom)
#reboot UDC to fix otg cd-rom and flash switch
udc_path = self.__drive.get_udc_path()
with open(udc_path) as file:
enabled = bool(file.read().strip())
if enabled:
#reset UDC to fix otg cd-rom and flash switch
try:
udc_path = self.__drive.get_udc_path()
with open(udc_path) as file:
enabled = bool(file.read().strip())
if enabled:
with open(udc_path, "w") as file:
file.write("\n")
with open(udc_path, "w") as file:
file.write("\n")
with open(udc_path, "w") as file:
file.write(sorted(os.listdir("/sys/class/udc"))[0])
file.write(sorted(os.listdir("/sys/class/udc"))[0])
except:
logger = get_logger(0)
logger.error("Can't reset UDC")
if self.__state.vd.rw:
await self.__state.vd.image.remount_rw(True)
self.__drive.set_image_path(self.__state.vd.image.path)
Expand All @@ -306,6 +326,86 @@ async def set_connected(self, connected: bool) -> None:

self.__state.vd.connected = connected

@aiotools.atomic_fg
async def make_image(self, zipped: bool) -> None:
#Note: img size >= 64M
def create_fat_image(img_size: int, file_img_path: str, source_dir: str, fat_type: int = 32, label: str = 'One-KVM'):
def add_directory_to_fat(fat: str, src_path: str, dst_path: str):
for item in os.listdir(src_path):
src_item_path = os.path.join(src_path, item)
dst_item_path = os.path.join(dst_path, item)

if os.path.isdir(src_item_path):
fat.makedir(dst_item_path)
add_directory_to_fat(fat, src_item_path, dst_item_path)
elif os.path.isfile(src_item_path):
with open(src_item_path, 'rb') as src_file:
fat.create(dst_item_path)
with fat.open(dst_item_path, 'wb') as dst_file:
dst_file.write(src_file.read())
print(file_img_path)
with open(file_img_path, 'wb') as f:
f.seek(img_size * 1024 *1024 - 1)
f.write(b'\0')
fat_file = pyfatfs.PyFat.PyFat()
try:
fat_file.mkfs(file_img_path, fat_type = fat_type, label = label)
except Exception as e:
get_logger(0).exception(f"Error making FAT Filesystem: {e}")
finally:
fat_file.close()
fat_handle = pyfatfs.PyFatFS.PyFatFS(file_img_path)
try:
add_directory_to_fat(fat_handle, source_dir, '/')
except Exception as e:
get_logger(0).exception(f"Error adding directory to FAT image: {e}")
finally:
fat_handle.close()

def extract_fat_image(file_img_path: str, output_dir: str):
try:
for root, dirs, files in os.walk(output_dir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
except Exception as e:
get_logger(0).exception(f"Error removing normal file or directory: {e}")
fat_handle = pyfatfs.PyFatFS.PyFatFS(file_img_path)
try:
def extract_directory(fat_handle, src_path: str, dst_path: str):
for entry in fat_handle.listdir(src_path):
src_item_path = os.path.join(src_path, entry)
dst_item_path = os.path.join(dst_path, entry)

if fat_handle.gettype(src_item_path) is pyfatfs.PyFatFS.ResourceType.directory:
os.makedirs(dst_item_path, exist_ok=True)
extract_directory(fat_handle, src_item_path, dst_item_path)
else:
with fat_handle.open(src_item_path, 'rb') as src_file:
with open(dst_item_path, 'wb') as dst_file:
dst_file.write(src_file.read())
extract_directory(fat_handle, '/', output_dir)
except Exception as e:
get_logger(0).exception(f"Error extracting FAT image: {e}")
finally:
fat_handle.close()

async with self.__state.busy():
msd_path = self.__msd_path
file_storage_path = os.path.join(msd_path, self.__normalfiles_path)
file_img_path = os.path.join(msd_path, self.__normalfiles_path + ".img")
img_size = self.__normalfiles_size
if zipped:
if not os.path.exists(file_storage_path):
os.makedirs(file_storage_path)
if os.path.exists(file_img_path):
os.remove(file_img_path)
create_fat_image(img_size, file_img_path, file_storage_path)
else:
if os.path.exists(file_img_path):
extract_fat_image(file_img_path, file_storage_path)

@contextlib.asynccontextmanager
async def read_image(self, name: str) -> AsyncGenerator[MsdFileReader, None]:
try:
Expand Down
1 change: 1 addition & 0 deletions testenv/requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ pyrad
types-PyYAML
types-aiofiles
luma.oled
pyfatfs
6 changes: 5 additions & 1 deletion testenv/v2-hdmiusb-rpi4.override.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@ kvmd:
noop: true

msd:
type: disabled
type: otg
remount_cmd: /bin/true
msd_path: /var/lib/kvmd/msd
normalfiles_path: NormalFiles
normalfiles_size: 64

streamer:
cmd:
Expand Down
20 changes: 19 additions & 1 deletion web/kvm/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,8 +596,18 @@
</div>
</td>
</tr>
<tr>
<td i18n="kvm_text84">文件内容：</td>
<td>
<div class="radio-box">
<input checked type="radio" id="msd-mode-radio-image" name="file-mode-radio" value="1">
<label for="msd-mode-radio-image" i18n="kvm_text90">ImageFiles</label>
<input type="radio" id="msd-mode-radio-file" name="file-mode-radio" value="0">
<label for="msd-mode-radio-file" i18n="kvm_text91">NormalFiles</label>
</div>
</td>
</tr>
</table>
<hr>
<div id="msd-storages"></div>
<hr>
<div class="buttons buttons-row">
Expand Down Expand Up @@ -674,6 +684,14 @@
<button class="row25" disabled id="msd-disconnect-button" i18n="kvm_text77">Disconnect</button>
<button class="row25" disabled id="msd-reset-button" i18n="kvm_text78">Reset</button>
</div>
<hr>
<div class="text"><b i18n="kvm_text85">Quick file transfer:</b><br><sub i18n="kvm_text86">&bull; Select NormalFiles tab to upload, package them and mount image</sub><br><sub i18n="kvm_text87">&bull; Disconnect MSD, unpackage it, select tab to download</sub><br></div>
<hr>
<div class="buttons buttons-row">
<button class="row50" id="msd-file-image-update-button" i18n="kvm_text88">Package files into image</button>
<button class="row50" id="msd-file-image-unzip-button" i18n="kvm_text89">Unpackage files from image</button>
</div>
<hr>
</div>
</li>
<li class="right" id="macro-dropdown"><a class="menu-button" href="#"><img class="led-gray" id="hid-recorder-led" src="/share/svg/led-gear.svg"><span i18n="kvm_text32">Macro</span></a>
Expand Down
26 changes: 25 additions & 1 deletion web/kvm/navbar-msd.pug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,14 @@ li(id="msd-dropdown" class="right feature-disabled")
label(for="msd-mode-radio-flash") Flash
td &nbsp;
+menu_switch_notable("msd-rw-switch", "Writable", false, false, "msd-rw-switch")
hr
tr
td(i18n="kvm_text84") 文件内容：
td
div(class="radio-box")
input(checked type="radio" id="msd-mode-radio-image" name="file-mode-radio" value="1")
label(for="msd-mode-radio-image" i18n="kvm_text90") ImageFiles
input(type="radio" id="msd-mode-radio-file" name="file-mode-radio" value="0")
label(for="msd-mode-radio-file" i18n="kvm_text91") NormalFiles
div(id="msd-storages")
hr
div(class="buttons buttons-row")
Expand Down Expand Up @@ -98,3 +105,20 @@ li(id="msd-dropdown" class="right feature-disabled")
button(disabled id="msd-connect-button" class="row50" i18n="kvm_text76") Connect drive to Server
button(disabled id="msd-disconnect-button" class="row25" i18n="kvm_text77") Disconnect
button(disabled id="msd-reset-button" class="row25" i18n="kvm_text78") Reset



hr
div(class="text")
b(i18n="kvm_text85") Quick file transfer:
br
sub(i18n="kvm_text86") &bull; Select NormalFiles tab to upload, package them and mount image
br
sub(i18n="kvm_text87") &bull; Disconnect MSD, unpackage it, select tab to download
br
hr
div(class="buttons buttons-row")
button(id="msd-file-image-update-button" class="row50" i18n="kvm_text88") Package files into image
button(id="msd-file-image-unzip-button" class="row50" i18n="kvm_text89") Unpackage files from image
hr

Loading

0 comments on commit b8ddf7c

Please sign in to comment.