Skip to content

ofDirectory::listDir performance improvement #7399

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dimitre wants to merge 7 commits into from
+45 −113
Closed

ofDirectory::listDir performance improvement #7399

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3159281
listdir and sort
dimitre Mar 17, 2023
771e9e5
revert example
dimitre Mar 17, 2023
04a92cc
fix
dimitre Mar 17, 2023
16da6f0
merge master
dimitre Sep 13, 2024
d89374f
more ofFile ideas
dimitre Sep 13, 2024
5823beb
ideas, changes
dimitre Sep 13, 2024
be5daaa
Merge branch 'master' into dir
dimitre Sep 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion libs/openFrameworks/communication/ofSerial.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ void ofSerial::buildDeviceList(){
ofDirectory dir("/dev");
int deviceCount = 0;
for(auto & entry: dir){
std::string deviceName = entry.getFileName();
std::string deviceName = entry.string();

//we go through the prefixes
for(auto & prefix: prefixMatch){
Expand Down
138 changes: 35 additions & 103 deletions libs/openFrameworks/utils/ofFileUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1123,7 +1123,7 @@ bool ofFile::doesFileExist(const fs::path & _path, bool bRelativeToData){
if(bRelativeToData){
path = ofToDataPath(path);
}
return !path.empty() && of::filesystem::exists(path);
return !path.empty() && fs::exists(path);
}

//------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -1380,51 +1380,54 @@ std::size_t ofDirectory::listDir(){
return 0;
}

fs::directory_iterator end_iter;
if ( fs::exists(myDir) && fs::is_directory(myDir)){
for (const auto & f : fs::directory_iterator{ myDir }) {
files.emplace_back(f.path(), ofFile::Reference);
for( fs::directory_iterator dir_iter(myDir) ; dir_iter != end_iter ; ++dir_iter){
files.emplace_back(dir_iter->path());
}
}else{
ofLogError("ofDirectory") << "listDir:() source directory does not exist: " << myDir ;
return 0;
}

if(!showHidden){
ofRemove(files, [](ofFile & file){
return file.isHidden();
});
}

// FIXME:
// if(!showHidden){
// ofRemove(files, [](fs::path & file){
// return file.isHidden();
// });
// }

if(!extensions.empty() && !ofContains(extensions, (string)"*")){
ofRemove(files, [&](ofFile & file){
return std::find(extensions.begin(), extensions.end(), ofToLower(file.getExtension())) == extensions.end();
});
}

// if(!extensions.empty() && !ofContains(extensions, (string)"*")){
// ofRemove(files, [&](ofFile & file){
// return std::find(extensions.begin(), extensions.end(), ofToLower(file.getExtension())) == extensions.end();
// });
// }

if(ofGetLogLevel() == OF_LOG_VERBOSE){
for(int i = 0; i < (int)size(); i++){
ofLogVerbose() << "\t" << getName(i);
}
ofLogVerbose() << "listed " << size() << " files in \"" << originalDirectory << "\"";
ofLogVerbose() << "listed " << size() << " files in " << originalDirectory;
}

return size();
}

//------------------------------------------------------------------------------------------------------------
string ofDirectory::getOriginalDirectory() const {
return ofPathToString(originalDirectory);
fs::path ofDirectory::getOriginalDirectory() const {
return originalDirectory;
}

//------------------------------------------------------------------------------------------------------------
string ofDirectory::getName(std::size_t position) const{
return files.at(position).getFileName();
fs::path ofDirectory::getName(std::size_t position) const{
return files.at(position);
}

//------------------------------------------------------------------------------------------------------------
string ofDirectory::getPath(std::size_t position) const{
return ofPathToString(originalDirectory / getName(position));
fs::path ofDirectory::getPath(std::size_t position) const{
return originalDirectory / getName(position);
}

//------------------------------------------------------------------------------------------------------------
Expand All @@ -1439,10 +1442,11 @@ ofFile ofDirectory::operator[](std::size_t position) const {
}

//------------------------------------------------------------------------------------------------------------
const vector<ofFile> & ofDirectory::getFiles() const{
const vector<fs::path> & ofDirectory::getFiles() const{
if(files.empty() && !myDir.empty()){
const_cast<ofDirectory*>(this)->listDir();
}
// FIXME
return files;
}

Expand All @@ -1456,94 +1460,22 @@ void ofDirectory::reset(){
close();
}

//------------------------------------------------------------------------------------------------------------
static bool natural(const ofFile& a, const ofFile& b) {
string aname = a.getBaseName(), bname = b.getBaseName();
int aint = ofToInt(aname), bint = ofToInt(bname);
if(ofToString(aint) == aname && ofToString(bint) == bname) {
return aint < bint;
} else {
return a < b;
}
}


//------------------------------------------------------------------------------------------------------------
struct StringSort{
fs::path path;
string basename;
int nameInt;
string stringInt;
};

//------------------------------------------------------------------------------------------------------------
static bool naturalStr(const StringSort& a, const StringSort& b) {
if(a.stringInt == a.basename && b.stringInt == b.basename) {
return a.nameInt < b.nameInt;
} else {
return a.path < b.path;
}
}

//------------------------------------------------------------------------------------------------------------
static bool byDate(const ofFile& a, const ofFile& b) {
auto ta = fs::last_write_time(a);
auto tb = fs::last_write_time(b);
return ta < tb;
}

//------------------------------------------------------------------------------------------------------------
void ofDirectory::sortByDate() {
if (files.empty() && !myDir.empty()) {
listDir();
}
ofSort(files, byDate);
std::sort(files.begin(), files.end(), [](const fs::path & a, const fs::path & b) {
return fs::last_write_time(a) < fs::last_write_time(b);
});
}

//------------------------------------------------------------------------------------------------------------
void ofDirectory::sort(const SortMode & mode){
if(files.empty() && !myDir.empty()){
listDir();
}

if( mode == ofDirectory::SORT_NATURAL ){
vector <StringSort> sort;
sort.reserve(files.size());

for( auto & f : files ){
StringSort ss;
ss.path = f.path();
ss.basename = f.getBaseName();
ss.nameInt = ofToInt(ss.basename);
ss.stringInt = ofToString(ss.nameInt);
sort.push_back(ss);
}

ofSort(sort, naturalStr);
files.clear();
files.reserve(sort.size());
for( auto & s : sort ){
files.emplace_back( s.path , ofFile::Reference);
}
}
else if(mode == ofDirectory::SORT_FAST){
std::vector <string> sort;
sort.reserve(files.size());

for( auto & f : files ){
string ss = f.getFileName();
sort.push_back(ss);
}

std::sort(sort.begin(), sort.end());
files.clear();
files.reserve(sort.size());
for( auto & s : sort ){
files.emplace_back( myDir / fs::path(s), ofFile::Reference);
}
}else if(mode == ofDirectory::SORT_BY_DATE){
sortByDate();
}
std::sort(files.begin(), files.end());
}

//------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -1675,22 +1607,22 @@ bool ofDirectory::operator>=(const ofDirectory & dir) const{
}

//------------------------------------------------------------------------------------------------------------
vector<ofFile>::const_iterator ofDirectory::begin() const{
return getFiles().begin();
vector<fs::path>::const_iterator ofDirectory::begin() const{
return files.begin();
}

//------------------------------------------------------------------------------------------------------------
vector<ofFile>::const_iterator ofDirectory::end() const{
vector<fs::path>::const_iterator ofDirectory::end() const{
return files.end();
}

//------------------------------------------------------------------------------------------------------------
vector<ofFile>::const_reverse_iterator ofDirectory::rbegin() const{
return getFiles().rbegin();
vector<fs::path>::const_reverse_iterator ofDirectory::rbegin() const{
return files.rbegin();
}

//------------------------------------------------------------------------------------------------------------
vector<ofFile>::const_reverse_iterator ofDirectory::rend() const{
vector<fs::path>::const_reverse_iterator ofDirectory::rend() const{
return files.rend();
}

Expand Down
18 changes: 9 additions & 9 deletions libs/openFrameworks/utils/ofFileUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1017,7 +1017,7 @@ class ofDirectory {
std::size_t listDir();

/// \returns the current path
std::string getOriginalDirectory() const;
of::filesystem::path getOriginalDirectory() const;

/// Get the filename at a given position in the directory contents
/// list, ie. "duck.jpg".
Expand All @@ -1028,7 +1028,7 @@ class ofDirectory {
/// listed directory contents.
/// \param position array index in the directory contents list
/// \returns file or directory name
std::string getName(std::size_t position) const;
of::filesystem::path getName(std::size_t position) const;

/// Get the full path of the file or directory at a given position in
/// the directory contents list.
Expand All @@ -1039,7 +1039,7 @@ class ofDirectory {
/// listed directory contents.
/// \param position array index in the directory contents list
/// \returns file or directory name including the current path
std::string getPath(std::size_t position) const;
of::filesystem::path getPath(std::size_t position) const;

/// Open an ofFile instance using the path a given position in the
/// directory contents list.
Expand All @@ -1063,7 +1063,7 @@ class ofDirectory {
/// Directory contents are automatically listed.
///
/// \returns vector of files in the directory
const std::vector<ofFile> & getFiles() const;
const std::vector<of::filesystem::path> & getFiles() const;

/// Access directory contents via th array operator.
///
Expand Down Expand Up @@ -1186,15 +1186,15 @@ class ofDirectory {
/// \returns true if the path was removed successfully
static bool removeDirectory(const of::filesystem::path & path, bool deleteIfNotEmpty, bool bRelativeToData = true);

std::vector<ofFile>::const_iterator begin() const;
std::vector<ofFile>::const_iterator end() const;
std::vector<ofFile>::const_reverse_iterator rbegin() const;
std::vector<ofFile>::const_reverse_iterator rend() const;
std::vector<of::filesystem::path>::const_iterator begin() const;
std::vector<of::filesystem::path>::const_iterator end() const;
std::vector<of::filesystem::path>::const_reverse_iterator rbegin() const;
std::vector<of::filesystem::path>::const_reverse_iterator rend() const;

of::filesystem::path myDir;
of::filesystem::path originalDirectory;
std::vector<std::string> extensions;
std::vector<ofFile> files;
std::vector<of::filesystem::path> files;
bool showHidden;
};

Expand Down