[Patch] Support for directory flags; a script to export everything from an HFS volume

[jamadden]

The Folder class has a "help me!" comment on the flags member: https://github.com/elliotnunn/machfs/blob/4f554286e6f5276a5b4d6d76f64fcecce31f1c26/machfs/directory.py#L337-L341

I needed to get the flags to be able to properly export all the information from an HFS volume (like getting custom icons to show), so I searched for the information and captured the flags. Here's a diff giving that (sorry, I haven't had the time to work up a proper PR):

--- orig_main.py    2023-06-12 06:44:47.054592462 -0500
+++ main.py 2023-06-12 06:01:50.353543249 -0500
@@ -259,12 +259,88 @@
             # print('\t', datarec)
             # print()

+            # Records described here:
+            # https://developer.apple.com/library/archive/documentation/mac/Files/Files-105.html#HEADING105-0
+            #
+            # Flags are interpreted as per https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-9A581/Finder.h
+            # /* Finder flags (finderFlags, fdFlags and frFlags) */
+            # enum {
+            #   kIsOnDesk       = 0x0001,     /* Files and folders (System 6) */
+            #   kColor          = 0x000E,     /* Files and folders */
+            #   kIsShared       = 0x0040,     /* Files only (Applications only) If */
+            #                                 /* clear, the application needs */
+            #                                 /* to write to its resource fork, */
+            #                                 /* and therefore cannot be shared */
+            #                                 /* on a server */
+            #   kHasNoINITs     = 0x0080,     /* Files only (Extensions/Control */
+            #                                 /* Panels only) */
+            #                                 /* This file contains no INIT resource */
+            #   kHasBeenInited  = 0x0100,     /* Files only.  Clear if the file */
+            #                                 /* contains desktop database resources */
+            #                                 /* ('BNDL', 'FREF', 'open', 'kind'...) */
+            #                                 /* that have not been added yet.  Set */
+            #                                 /* only by the Finder. */
+            #                                 /* Reserved for folders */
+            #   kHasCustomIcon  = 0x0400,     /* Files and folders */
+            #   kIsStationery   = 0x0800,     /* Files only */
+            #   kNameLocked     = 0x1000,     /* Files and folders */
+            #   kHasBundle      = 0x2000,     /* Files only */
+            #   kIsInvisible    = 0x4000,     /* Files and folders */
+            #   kIsAlias        = 0x8000      /* Files only */
+            # };
+
+            # /* Extended flags (extendedFinderFlags, fdXFlags and frXFlags) */
+            # enum {
+            #   kExtendedFlagsAreInvalid    = 0x8000, /* The other extended flags */
+            #                                         /* should be ignored */
+            #   kExtendedFlagHasCustomBadge = 0x0100, /* The file or folder has a */
+            #                                         /* badge resource */
+            #   kExtendedFlagHasRoutingInfo = 0x0004  /* The file contains routing */
+            #                                         /* info resource */
+            # };
             if datatype == 'dir':
+                # cdrDirRec:                    {directory record}
+                #   dirFlags:       Integer;    {directory flags}
+                #    dirVal:        Integer;    {directory valence}
+                #    dirDirID:      LongInt;    {directory ID}
+                #    dirCrDat:      LongInt;    {date and time of creation}
+                #    dirMdDat:      LongInt;    {date and time of last modification}
+                #    dirBkDat:      LongInt;    {date and time of last backup}
+                #    dirUsrInfo:    DInfo;      {Finder information}
+                #    dirFndrInfo:   DXInfo;     {additional Finder information}
+                #    dirResrv:      ARRAY[1..4] OF LongInt;
+
                 dirFlags, dirVal, dirDirID, dirCrDat, dirMdDat, dirBkDat, dirUsrInfo, dirFndrInfo \
                 = struct.unpack_from('>HHLLLL16s16s', datarec)

+                # dirUsrInfo is a DInfo.
+                # https://developer.apple.com/library/archive/documentation/mac/Toolbox/Toolbox-466.html#HEADING466-0
+                # TYPE DInfo =
+                # RECORD
+                #    frRect:     Rect;    {folder's window rectangle}
+                #    frFlags:    Integer; {flags}
+                #    frLocation: Point;   {folder's location in window}
+                #    frView:     Integer; {folder's view}
+                # END;
+                _x, _y, _xx, _yy, flags, x, y, view = struct.unpack_from('>HHHHHHHH', dirUsrInfo)
+
+                # dirFndrInfo is a DXInfo:
+                # TYPE  DXInfo =
+                #       RECORD
+                #          frScroll:      Point;      {scroll position}
+                #          frOpenChain:   LongInt;    {directory ID chain of open }
+                #                                     { folders}
+                #          frScript:      SignedByte; {script flag and code}
+                #          frXFlags:      SignedByte; {reserved}
+                #          frComment:     Integer;    {comment ID}
+                #          frPutAway:     LongInt;    {home directory ID}
+                #       END;
+
                 f = Folder()
                 cnids[dirDirID] = f
+                f.flags = flags
+                f.x, f.y = x, y
+                f.view = view
                 childlist.append((ckrParID, ckrCName, f))

                 f.crdate, f.mddate, f.bkdate = dirCrDat, dirMdDat, dirBkDat

With that, I am able to use the following script on a macOS machine to export an HFS volume to a set of folders where the data and resource forks are preserved, as is the color label, position, and custom icons. Unfortunately, while the folder icons and positions show up correctly when accessed by macOS (Ventura) whether on a local disk or served over SMB, they do not show up in a classic MacOS 9.1 client when served over AFP via netatalk from the same volume that macOS Ventura was accessing them over SMB.

# -*- coding: utf-8 -*-
"""
For copying a HFS volume.
"""
import sys
import subprocess

from datetime import datetime as DateTime
from pathlib import Path

from machfs import Volume
from machfs import Folder
from machfs import File
from machfs.directory import AbstractFolder

class FixerMixin:
    def fixup(self):
        for name, val in self.items():
            val.__name__ = name
            val.__parent__ = self
            if isinstance(val, Folder):
                val.__class__ = SmartFolder
                val.fixup()
            else:
                assert isinstance(val, File)
                val.__class__ = SmartFile

AbstractFolder.fixup = FixerMixin.fixup

class SmartVolume(FixerMixin, Volume):

    @property
    def __name__(self):
        return self.name

    @property
    def __path__(self):
        return Path(self.__name__)

    def read(self, *args):
        super().read(*args)
        self.fixup()

class SmartFolder(Folder):
    __parent__ = None
    __name__ = None
    type = b'????'
    creator = b'????'

    @property
    def __path__(self):
        return self.__parent__.__path__ / self.__name__

    def __repr__(self):
        return '<Folder %r at %r>' % (
            self.__name__,
            self.__path__
        )

class SmartFile(File):
    __name__ = None
    __parent__ = None
    @property
    def __path__(self):
        return self.__parent__.__path__ / self.__name__.replace('/', ':').replace('\x00', '_')

MAC_EPOCH = DateTime(1904, 1, 1)

def _date_str_for_SetFile(date):
    date = DateTime.fromtimestamp(date + MAC_EPOCH.timestamp())
    #  "mm/dd/[yy]yy [hh:mm:[:ss] [AM | PM]]"
    hour = date.hour
    if hour >= 12:
        am_pm = 'PM'
        hour = hour - 12
    else:
        am_pm = 'AM'
        if hour == 0:
            hour = 12

    return "%s/%s/%s %s:%s %s" % (
        date.month, date.day, date.year,
        hour, date.minute, am_pm
    )

def _set_hfs_attribs(file:File, dest_file:Path, verbose=True):
    creator = file.creator
    type = file.type
    crdate = _date_str_for_SetFile(file.crdate) if file.crdate else None
    mddate = _date_str_for_SetFile(file.mddate) if file.mddate else None
    folder_icon = False

    if creator == b'\x00\x00\x00\x00' and type == b'\x00\x00\x00\x00':
        if file.__name__ == 'Icon\r':
            # macOS X wants these types; classic MacOS 9 leaves them at 0.
            # It doesn't seem to make a difference on where/if they are displayed.
            creator = b'MACS'
            type = b'icon'
            folder_icon = True
        else:
            creator = type = b''

    cmd = [
        'SetFile',
    ]
    if crdate:
        cmd.extend([
            '-d', crdate
        ])

    if mddate:
        cmd.extend([
            '-m', mddate
        ])

    if creator != b'????':
        cmd.extend([
            '-c',
            creator.decode('mac_roman'),
        ])
    if type != b'????':
        cmd.extend([
            '-t',
            type.decode('mac_roman'),
        ])

    flags_to_set = set()
    if folder_icon:
        flags_to_set = {'V', 'C'}

        # Shouldn't need to do this anymore since we're
        # setting the folder directly, right?
        subprocess.check_call([
            'SetFile',
            '-a', 'CINE',
            dest_file.parent,
        ])

    if file.flags & 0x000E:
        # Has a color assigned to it
        color = (file.flags & 0x000E)
        color = color >> 1
        if verbose:
            print('Color', repr(dest_file), color)
        # Applescript to set label
        # Or we could communicate directly with the finder using
        # the scripting bridge.
        script = """
        on run argv
            tell application "Finder"
                set theFile to POSIX file (item 1 of argv) as alias
                set labelIdx to (item 2 of argv as number)
                set label index of theFile to labelIdx
            end tell
        end run
        """
        subprocess.check_call([
            'osascript', '-e', script,
            dest_file, str(color)
        ])

    if (file.x > 0 or file.y > 0) and (file.x <= 32768 and file.y <= 32768):
        if verbose:
            print('Location', repr(dest_file), file.x, file.y)
        script = """
        on run argv
            tell application "Finder"
                set theFile to POSIX file (item 1 of argv) as alias
                set px to (item 2 of argv as number)
                set py to (item 3 of argv as number)
                set position of theFile to {px, py}
            end tell
        end run
        """
        subprocess.check_call([
            'osascript', '-e', script,
            dest_file, str(file.x), str(file.y),
        ])

    for mask, attr in (
        (0x0001, 'D'), # On desk
        (0x0080, 'N'), # No init
        (0x0100, 'I'), # initted
        (0x0400, 'C'), # Custom icon
        (0x2000, 'B'), # has bundle
        (0x4000, 'V'), # invisible
        (0x8000, 'A'), # alias
    ):
        if file.flags & mask:
            flags_to_set.add(attr)

    if flags_to_set:
        cmd.append('-a')
        cmd.append(''.join(flags_to_set))

    cmd.append(dest_file)
    if verbose:
        print(cmd, file.creator, file.type, 'flags=0x' + hex(file.flags), flags_to_set)
    subprocess.check_call(cmd)

def cp_file(file:SmartFile, root_dir:Path):

    dest_file = root_dir / file.__path__
    dest_file.parent.mkdir(parents=True, exist_ok=True)
    dest_file.write_bytes(file.data)
    if file.rsrc:
        rfork = dest_file / '..namedfork' / 'rsrc'
        rfork.write_bytes(file.rsrc)

    _set_hfs_attribs(file, dest_file, verbose=False)

def mk_dir(item:SmartFolder, root_dir:Path):
    dest_file = root_dir / item.__path__
    dest_file.mkdir(exist_ok=True, parents=True)
    _set_hfs_attribs(item, dest_file)

def read_volume(iso_path:Path):
    data = iso_path.read_bytes()
    vol = SmartVolume()
    vol.read(data)
    return vol

def main(args=None):
    args = args or sys.argv[1:]
    iso = Path(args[0])
    dest = Path(args[1])

    vol = read_volume(iso)

    for _, item in vol.iter_paths():
        #print(repr(item.__name__), ' <- at ->', repr(item.__path__), type(item))
        if hasattr(item, 'aliastarget') and item.aliastarget is not None:
            print('\tAlias:', repr(item.aliastarget))
        if isinstance(item, Folder):
            mk_dir(item, dest)
        if isinstance(item, File) and item.aliastarget is None:
            cp_file(item, dest)

if __name__ == '__main__':
    main()

[jamadden]

Unfortunately, while the folder icons and positions show up correctly when accessed by macOS (Ventura) whether on a local disk or served over SMB, they do not show up in a classic MacOS 9.1 client when served over AFP via netatalk from the same volume that macOS Ventura was accessing them over SMB.

This turns out to be some difference in the way the SMB and AFP protocols are handling things. Copying to an AFP share from Ventura produces viewable icans when looked at from another AFP client, but not when looked at from an SMB client, and vice versa (copying to SMB looks fine to SMB clients, but not to AFP clients).