From ba060bfe965d193d74a78178841f838ea0c4d8cf Mon Sep 17 00:00:00 2001 From: David Han Sze Chuen <182644+kayhadrin@users.noreply.github.com> Date: Sat, 3 May 2025 18:07:14 +0000 Subject: [PATCH 1/2] duperemove: create duperemove package Makefile copied from a similar PR in the OpenWRT repo by @ktgeek See https://github.com/openwrt/packages/pull/26206 --- duperemove/Makefile | 103 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 duperemove/Makefile diff --git a/duperemove/Makefile b/duperemove/Makefile new file mode 100644 index 00000000..b58a716d --- /dev/null +++ b/duperemove/Makefile @@ -0,0 +1,103 @@ +# +# This is free software, licensed under the GNU General Public License v2. +# See /LICENSE for more information. +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=duperemove +PKG_VERSION:=0.15.1 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/markfasheh/duperemove/tar.gz/v$(PKG_VERSION)? +PKG_HASH:=68cc28f5aa43fa2034e512f7b22cf5282ce2b0319b4e1061f7cdf55cc134273b + +PKG_MAINTAINER:=Keith Garner +PKG_LICENSE:=GPL-2.0 +PKG_LICENSE_FILES:=LICENSE + +# Set variables used in the source's Makefile +export VERSION="$(PKG_VERSION)" +export IS_RELEASE=1 + +include $(INCLUDE_DIR)/package.mk + +define Package/duperemove + SECTION:=utils + CATEGORY:=Utilities + SUBMENU:=Filesystem + TITLE:=Duplicate file removal tool + URL:=https://github.com/markfasheh/duperemove + DEPENDS:= \ + +libsqlite3 \ + +glib2 \ + +libxxhash \ + +libuuid \ + +libmount \ + +libblkid \ + +libbsd \ + +libatomic +endef + +define Package/duperemove/description + Duperemove is a simple tool for finding duplicated extents and submitting + them for deduplication. When given a list of files it will hash their + contents on an extent by extent basis and compare those hashes to each + other, finding and categorizing extents that match each other. Optionally, + a per-block hash can be applied for further duplication lookup. When given + the -d option, duperemove will submit those extents for deduplication using + the Linux kernel FIDEDUPRANGE ioctl. +endef + +define Package/duperemove/install + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/duperemove $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/hashstats $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/btrfs-extent-same $(1)/usr/bin +endef + +# define Build/Configure +# $(call Build/Configure/Default,, \ +# UNAME_S="Linux" \ +# UNAME_R="$(LINUX_VERSION)" \ +# UNAME_M="$(ARCH)" \ +# ) +# # Copy kernel headers to the staging directory +# $(CP) /usr/include/linux $(STAGING_DIR)/opt/include +# endef + +# define Build/InstallDev +# $(INSTALL_DIR) \ +# $(1)/opt/include \ +# $(1)/opt/lib + +# $(CP) \ +# $(PKG_INSTALL_DIR)/opt/include/* \ +# $(1)/opt/include/ + +# $(CP) \ +# $(PKG_INSTALL_DIR)/opt/lib/{pkgconfig,*.so*} \ +# $(1)/opt/lib/ +# endef + + +# define Build/Configure +# $(call Build/Configure/Default,, \ +# UNAME_S="Linux" \ +# UNAME_R="$(LINUX_VERSION)" \ +# UNAME_M="$(ARCH)" \ +# ) +# mkdir -p $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux +# $(CP) \ +# $(LINUX_DIR)/include/linux/compiler.h \ +# $(LINUX_DIR)/include/$(LINUX_UAPI_DIR)linux/atm*.h \ +# $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux/ + +# # Kernel 4.14.9+ only, ignore the exit status of cp in case the file +# # doesn't exits +# -$(CP) $(LINUX_DIR)/include/linux/compiler_types.h \ +# $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux/ +# endef + +$(eval $(call BuildPackage,duperemove)) \ No newline at end of file From dffcf1ed7ad42f4a6825bd0ccbbc96efdfdef66a Mon Sep 17 00:00:00 2001 From: David Han Sze Chuen <182644+kayhadrin@users.noreply.github.com> Date: Mon, 5 May 2025 12:12:04 +0000 Subject: [PATCH 2/2] duperemove: document my hacky way to compile this --- duperemove/Makefile | 98 ++++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/duperemove/Makefile b/duperemove/Makefile index b58a716d..47c41a12 100644 --- a/duperemove/Makefile +++ b/duperemove/Makefile @@ -29,6 +29,7 @@ define Package/duperemove SUBMENU:=Filesystem TITLE:=Duplicate file removal tool URL:=https://github.com/markfasheh/duperemove + # This "pkg-config" is reportedly needed by duperemove but it seems to compile fine without it. DEPENDS:= \ +libsqlite3 \ +glib2 \ @@ -57,47 +58,80 @@ define Package/duperemove/install $(INSTALL_BIN) $(PKG_BUILD_DIR)/btrfs-extent-same $(1)/usr/bin endef -# define Build/Configure -# $(call Build/Configure/Default,, \ -# UNAME_S="Linux" \ -# UNAME_R="$(LINUX_VERSION)" \ -# UNAME_M="$(ARCH)" \ -# ) -# # Copy kernel headers to the staging directory -# $(CP) /usr/include/linux $(STAGING_DIR)/opt/include -# endef +# Add "/usr/include" in the include path to have access to "btrfs.h" +# These headers are not included in the Entware toolchain at $(STAGING_DIR)/opt/include +TARGET_CFLAGS += \ + -I/usr/include -# define Build/InstallDev -# $(INSTALL_DIR) \ -# $(1)/opt/include \ -# $(1)/opt/lib - -# $(CP) \ -# $(PKG_INSTALL_DIR)/opt/include/* \ -# $(1)/opt/include/ - -# $(CP) \ -# $(PKG_INSTALL_DIR)/opt/lib/{pkgconfig,*.so*} \ -# $(1)/opt/lib/ -# endef +# Added these flags to enable linker debugging +TARGET_CFLAGS += \ + -Xlinker --verbose +# Adding another path for libintl because the linker wrongly assumes that it's in $(STAGING_DIR)/opt/lib +TARGET_LDFLAGS += \ + -L$(STAGING_DIR)/opt/lib/libintl-full/lib +# The linker couldn't also find libiconv, so I added the hacky solution below. +# I tried using: -L$(STAGING_DIR)/opt/lib/libiconv-full/lib +# But it didn't work because gcc wouldn't use it during the linking process. +# It gets used for linking other shared libraries, but not libiconv. +# +# So I tried using "rpath" instead and it seems to work, but I had issues to get a proper path. +# I noticed that if rpath starts with "/" (e.g. /home/...), then the generated gcc command will treat it as a relative path. +# If it starts with a non-slash character (e.g. example/opt/lib/...), it'll be treated as an absolute path, which is not what I want. +# I'm not sure why it's the case. Maybe it's a custom behavior of the compiled gcc from Entware? +# +# I previously tried using $ORIGIN in various ways but couldn't find a way to escape it properly. +# Either the generated gcc command fails because Make doesn't know how to escape it when generating intermediate bash commands, or the final command is incorrect. +# E.g. Those don't work: +# -Wl,-rpath,'$$$$ORIGIN'/libiconv-full/lib +# -Wl,-rpath,"'"'$$'ORIGIN"'"/libiconv-full/lib +# "'"'$'ORIGIN"'" + +# Some dirname magic to get the relative path to the staging directory +PKG_REL_DIR := \ + $(shell dirname \ + $(patsubst \ + $(shell dirname \ + $(shell dirname $(PKG_BUILD_DIR)) \ + )/%,%,$(PKG_BUILD_DIR) \ + ) \ + ) +TARGET_LDFLAGS += \ + -Wl,-rpath,/../$(PKG_REL_DIR)/opt/lib/libiconv-full/lib + +# DEBUGGING info +$(info PKG_BUILD_DIR: $(PKG_BUILD_DIR)) +# E.g. PKG_BUILD_DIR: /home/me/Entware/build_dir/target-x86_64_glibc-2.33/duperemove-0.15.1 +$(info TARGET_LDFLAGS = $(TARGET_LDFLAGS)) +# E.g. TARGET_LDFLAGS = -Wl,--dynamic-linker=/opt/lib/ld-linux-x86-64.so.2 -Wl,-rpath=/opt/lib -L/home/me/Entware/staging_dir/toolchain-x86_64_gcc-12.3.0_glibc-2.33/usr/lib -L/home/me/Entware/staging_dir/toolchain-x86_64_gcc-12.3.0_glibc-2.33/lib -fuse-ld=bfd -L/ + +# I tried other alternatives like copying or creating simlinks to the shared libraries but it felt too hacky. Maybe it's the right way to do it? # define Build/Configure # $(call Build/Configure/Default,, \ # UNAME_S="Linux" \ # UNAME_R="$(LINUX_VERSION)" \ # UNAME_M="$(ARCH)" \ # ) -# mkdir -p $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux -# $(CP) \ -# $(LINUX_DIR)/include/linux/compiler.h \ -# $(LINUX_DIR)/include/$(LINUX_UAPI_DIR)linux/atm*.h \ -# $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux/ - -# # Kernel 4.14.9+ only, ignore the exit status of cp in case the file -# # doesn't exits -# -$(CP) $(LINUX_DIR)/include/linux/compiler_types.h \ -# $(PKG_BUILD_DIR)/pppd/plugins/pppoatm/linux/ +# echo ##################################################### +# echo PKG_BUILD_DIR: $(PKG_BUILD_DIR) +# echo STAGING_DIR: $(STAGING_DIR) +# echo CC: $(CC) +# echo CFLAGS: $(CFLAGS) +# echo EXTRA_CFLAGS: $(EXTRA_CFLAGS) +# echo LDFLAGS: $(LDFLAGS) +# echo TARGET_CFLAGS: $(TARGET_CFLAGS) +# echo TARGET_LDFLAGS: $(TARGET_LDFLAGS) +# echo ##################################################### +# +# # echo ##################################################### +# # echo # Copy kernel headers to the staging directory +# # echo ##################################################### +# # $(CP) /usr/include/linux $(STAGING_DIR)/opt/include +# +# # Create links to some shared libraries +# # I tried using '$ORIGIN' with LDFLAGS but couldn't find a way to escape it properly to please Makefile and the generated gcc command. +# # ln -sf $(STAGING_DIR)/opt/lib/libiconv-full/lib/* $(STAGING_DIR)/opt/lib # endef $(eval $(call BuildPackage,duperemove)) \ No newline at end of file