|
@@ -111,6 +111,17 @@ static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
|
|
|
return dentry_hashtable + hash_32(hash, d_hash_shift);
|
|
|
}
|
|
|
|
|
|
+#define IN_LOOKUP_SHIFT 10
|
|
|
+static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
|
|
|
+
|
|
|
+static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
|
|
|
+ unsigned int hash)
|
|
|
+{
|
|
|
+ hash += (unsigned long) parent / L1_CACHE_BYTES;
|
|
|
+ return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
/* Statistics gathering. */
|
|
|
struct dentry_stat_t dentry_stat = {
|
|
|
.age_limit = 45,
|
|
@@ -761,6 +772,8 @@ repeat:
|
|
|
/* Slow case: now with the dentry lock held */
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
+ WARN_ON(d_in_lookup(dentry));
|
|
|
+
|
|
|
/* Unreachable? Get rid of it */
|
|
|
if (unlikely(d_unhashed(dentry)))
|
|
|
goto kill_it;
|
|
@@ -1746,6 +1759,7 @@ type_determined:
|
|
|
static void __d_instantiate(struct dentry *dentry, struct inode *inode)
|
|
|
{
|
|
|
unsigned add_flags = d_flags_for_inode(inode);
|
|
|
+ WARN_ON(d_in_lookup(dentry));
|
|
|
|
|
|
spin_lock(&dentry->d_lock);
|
|
|
hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
|
|
@@ -1775,11 +1789,11 @@ void d_instantiate(struct dentry *entry, struct inode * inode)
|
|
|
{
|
|
|
BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
|
|
|
if (inode) {
|
|
|
+ security_d_instantiate(entry, inode);
|
|
|
spin_lock(&inode->i_lock);
|
|
|
__d_instantiate(entry, inode);
|
|
|
spin_unlock(&inode->i_lock);
|
|
|
}
|
|
|
- security_d_instantiate(entry, inode);
|
|
|
}
|
|
|
EXPORT_SYMBOL(d_instantiate);
|
|
|
|
|
@@ -1796,6 +1810,7 @@ int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
|
|
|
{
|
|
|
BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
|
|
|
|
|
|
+ security_d_instantiate(entry, inode);
|
|
|
spin_lock(&inode->i_lock);
|
|
|
if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) {
|
|
|
spin_unlock(&inode->i_lock);
|
|
@@ -1804,7 +1819,6 @@ int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
|
|
|
}
|
|
|
__d_instantiate(entry, inode);
|
|
|
spin_unlock(&inode->i_lock);
|
|
|
- security_d_instantiate(entry, inode);
|
|
|
|
|
|
return 0;
|
|
|
}
|
|
@@ -1878,6 +1892,7 @@ static struct dentry *__d_obtain_alias(struct inode *inode, int disconnected)
|
|
|
goto out_iput;
|
|
|
}
|
|
|
|
|
|
+ security_d_instantiate(tmp, inode);
|
|
|
spin_lock(&inode->i_lock);
|
|
|
res = __d_find_any_alias(inode);
|
|
|
if (res) {
|
|
@@ -1900,13 +1915,10 @@ static struct dentry *__d_obtain_alias(struct inode *inode, int disconnected)
|
|
|
hlist_bl_unlock(&tmp->d_sb->s_anon);
|
|
|
spin_unlock(&tmp->d_lock);
|
|
|
spin_unlock(&inode->i_lock);
|
|
|
- security_d_instantiate(tmp, inode);
|
|
|
|
|
|
return tmp;
|
|
|
|
|
|
out_iput:
|
|
|
- if (res && !IS_ERR(res))
|
|
|
- security_d_instantiate(res, inode);
|
|
|
iput(inode);
|
|
|
return res;
|
|
|
}
|
|
@@ -1975,28 +1987,36 @@ EXPORT_SYMBOL(d_obtain_root);
|
|
|
struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
|
|
|
struct qstr *name)
|
|
|
{
|
|
|
- struct dentry *found;
|
|
|
- struct dentry *new;
|
|
|
+ struct dentry *found, *res;
|
|
|
|
|
|
/*
|
|
|
* First check if a dentry matching the name already exists,
|
|
|
* if not go ahead and create it now.
|
|
|
*/
|
|
|
found = d_hash_and_lookup(dentry->d_parent, name);
|
|
|
- if (!found) {
|
|
|
- new = d_alloc(dentry->d_parent, name);
|
|
|
- if (!new) {
|
|
|
- found = ERR_PTR(-ENOMEM);
|
|
|
- } else {
|
|
|
- found = d_splice_alias(inode, new);
|
|
|
- if (found) {
|
|
|
- dput(new);
|
|
|
- return found;
|
|
|
- }
|
|
|
- return new;
|
|
|
+ if (found) {
|
|
|
+ iput(inode);
|
|
|
+ return found;
|
|
|
+ }
|
|
|
+ if (d_in_lookup(dentry)) {
|
|
|
+ found = d_alloc_parallel(dentry->d_parent, name,
|
|
|
+ dentry->d_wait);
|
|
|
+ if (IS_ERR(found) || !d_in_lookup(found)) {
|
|
|
+ iput(inode);
|
|
|
+ return found;
|
|
|
}
|
|
|
+ } else {
|
|
|
+ found = d_alloc(dentry->d_parent, name);
|
|
|
+ if (!found) {
|
|
|
+ iput(inode);
|
|
|
+ return ERR_PTR(-ENOMEM);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ res = d_splice_alias(inode, found);
|
|
|
+ if (res) {
|
|
|
+ dput(found);
|
|
|
+ return res;
|
|
|
}
|
|
|
- iput(inode);
|
|
|
return found;
|
|
|
}
|
|
|
EXPORT_SYMBOL(d_add_ci);
|
|
@@ -2363,17 +2383,194 @@ void d_rehash(struct dentry * entry)
|
|
|
}
|
|
|
EXPORT_SYMBOL(d_rehash);
|
|
|
|
|
|
+static inline unsigned start_dir_add(struct inode *dir)
|
|
|
+{
|
|
|
+
|
|
|
+ for (;;) {
|
|
|
+ unsigned n = dir->i_dir_seq;
|
|
|
+ if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
|
|
|
+ return n;
|
|
|
+ cpu_relax();
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static inline void end_dir_add(struct inode *dir, unsigned n)
|
|
|
+{
|
|
|
+ smp_store_release(&dir->i_dir_seq, n + 2);
|
|
|
+}
|
|
|
+
|
|
|
+static void d_wait_lookup(struct dentry *dentry)
|
|
|
+{
|
|
|
+ if (d_in_lookup(dentry)) {
|
|
|
+ DECLARE_WAITQUEUE(wait, current);
|
|
|
+ add_wait_queue(dentry->d_wait, &wait);
|
|
|
+ do {
|
|
|
+ set_current_state(TASK_UNINTERRUPTIBLE);
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+ schedule();
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
+ } while (d_in_lookup(dentry));
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+struct dentry *d_alloc_parallel(struct dentry *parent,
|
|
|
+ const struct qstr *name,
|
|
|
+ wait_queue_head_t *wq)
|
|
|
+{
|
|
|
+ unsigned int len = name->len;
|
|
|
+ unsigned int hash = name->hash;
|
|
|
+ const unsigned char *str = name->name;
|
|
|
+ struct hlist_bl_head *b = in_lookup_hash(parent, hash);
|
|
|
+ struct hlist_bl_node *node;
|
|
|
+ struct dentry *new = d_alloc(parent, name);
|
|
|
+ struct dentry *dentry;
|
|
|
+ unsigned seq, r_seq, d_seq;
|
|
|
+
|
|
|
+ if (unlikely(!new))
|
|
|
+ return ERR_PTR(-ENOMEM);
|
|
|
+
|
|
|
+retry:
|
|
|
+ rcu_read_lock();
|
|
|
+ seq = smp_load_acquire(&parent->d_inode->i_dir_seq) & ~1;
|
|
|
+ r_seq = read_seqbegin(&rename_lock);
|
|
|
+ dentry = __d_lookup_rcu(parent, name, &d_seq);
|
|
|
+ if (unlikely(dentry)) {
|
|
|
+ if (!lockref_get_not_dead(&dentry->d_lockref)) {
|
|
|
+ rcu_read_unlock();
|
|
|
+ goto retry;
|
|
|
+ }
|
|
|
+ if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
|
|
|
+ rcu_read_unlock();
|
|
|
+ dput(dentry);
|
|
|
+ goto retry;
|
|
|
+ }
|
|
|
+ rcu_read_unlock();
|
|
|
+ dput(new);
|
|
|
+ return dentry;
|
|
|
+ }
|
|
|
+ if (unlikely(read_seqretry(&rename_lock, r_seq))) {
|
|
|
+ rcu_read_unlock();
|
|
|
+ goto retry;
|
|
|
+ }
|
|
|
+ hlist_bl_lock(b);
|
|
|
+ if (unlikely(parent->d_inode->i_dir_seq != seq)) {
|
|
|
+ hlist_bl_unlock(b);
|
|
|
+ rcu_read_unlock();
|
|
|
+ goto retry;
|
|
|
+ }
|
|
|
+ rcu_read_unlock();
|
|
|
+ /*
|
|
|
+ * No changes for the parent since the beginning of d_lookup().
|
|
|
+ * Since all removals from the chain happen with hlist_bl_lock(),
|
|
|
+ * any potential in-lookup matches are going to stay here until
|
|
|
+ * we unlock the chain. All fields are stable in everything
|
|
|
+ * we encounter.
|
|
|
+ */
|
|
|
+ hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
|
|
|
+ if (dentry->d_name.hash != hash)
|
|
|
+ continue;
|
|
|
+ if (dentry->d_parent != parent)
|
|
|
+ continue;
|
|
|
+ if (d_unhashed(dentry))
|
|
|
+ continue;
|
|
|
+ if (parent->d_flags & DCACHE_OP_COMPARE) {
|
|
|
+ int tlen = dentry->d_name.len;
|
|
|
+ const char *tname = dentry->d_name.name;
|
|
|
+ if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
|
|
|
+ continue;
|
|
|
+ } else {
|
|
|
+ if (dentry->d_name.len != len)
|
|
|
+ continue;
|
|
|
+ if (dentry_cmp(dentry, str, len))
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ dget(dentry);
|
|
|
+ hlist_bl_unlock(b);
|
|
|
+ /* somebody is doing lookup for it right now; wait for it */
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
+ d_wait_lookup(dentry);
|
|
|
+ /*
|
|
|
+ * it's not in-lookup anymore; in principle we should repeat
|
|
|
+ * everything from dcache lookup, but it's likely to be what
|
|
|
+ * d_lookup() would've found anyway. If it is, just return it;
|
|
|
+ * otherwise we really have to repeat the whole thing.
|
|
|
+ */
|
|
|
+ if (unlikely(dentry->d_name.hash != hash))
|
|
|
+ goto mismatch;
|
|
|
+ if (unlikely(dentry->d_parent != parent))
|
|
|
+ goto mismatch;
|
|
|
+ if (unlikely(d_unhashed(dentry)))
|
|
|
+ goto mismatch;
|
|
|
+ if (parent->d_flags & DCACHE_OP_COMPARE) {
|
|
|
+ int tlen = dentry->d_name.len;
|
|
|
+ const char *tname = dentry->d_name.name;
|
|
|
+ if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
|
|
|
+ goto mismatch;
|
|
|
+ } else {
|
|
|
+ if (unlikely(dentry->d_name.len != len))
|
|
|
+ goto mismatch;
|
|
|
+ if (unlikely(dentry_cmp(dentry, str, len)))
|
|
|
+ goto mismatch;
|
|
|
+ }
|
|
|
+ /* OK, it *is* a hashed match; return it */
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+ dput(new);
|
|
|
+ return dentry;
|
|
|
+ }
|
|
|
+ /* we can't take ->d_lock here; it's OK, though. */
|
|
|
+ new->d_flags |= DCACHE_PAR_LOOKUP;
|
|
|
+ new->d_wait = wq;
|
|
|
+ hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
|
|
|
+ hlist_bl_unlock(b);
|
|
|
+ return new;
|
|
|
+mismatch:
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+ dput(dentry);
|
|
|
+ goto retry;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(d_alloc_parallel);
|
|
|
+
|
|
|
+void __d_lookup_done(struct dentry *dentry)
|
|
|
+{
|
|
|
+ struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent,
|
|
|
+ dentry->d_name.hash);
|
|
|
+ hlist_bl_lock(b);
|
|
|
+ dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
|
|
|
+ __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
|
|
|
+ wake_up_all(dentry->d_wait);
|
|
|
+ dentry->d_wait = NULL;
|
|
|
+ hlist_bl_unlock(b);
|
|
|
+ INIT_HLIST_NODE(&dentry->d_u.d_alias);
|
|
|
+ INIT_LIST_HEAD(&dentry->d_lru);
|
|
|
+}
|
|
|
+EXPORT_SYMBOL(__d_lookup_done);
|
|
|
|
|
|
/* inode->i_lock held if inode is non-NULL */
|
|
|
|
|
|
static inline void __d_add(struct dentry *dentry, struct inode *inode)
|
|
|
{
|
|
|
+ struct inode *dir = NULL;
|
|
|
+ unsigned n;
|
|
|
+ spin_lock(&dentry->d_lock);
|
|
|
+ if (unlikely(d_in_lookup(dentry))) {
|
|
|
+ dir = dentry->d_parent->d_inode;
|
|
|
+ n = start_dir_add(dir);
|
|
|
+ __d_lookup_done(dentry);
|
|
|
+ }
|
|
|
if (inode) {
|
|
|
- __d_instantiate(dentry, inode);
|
|
|
+ unsigned add_flags = d_flags_for_inode(inode);
|
|
|
+ hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
|
|
|
+ raw_write_seqcount_begin(&dentry->d_seq);
|
|
|
+ __d_set_inode_and_type(dentry, inode, add_flags);
|
|
|
+ raw_write_seqcount_end(&dentry->d_seq);
|
|
|
+ __fsnotify_d_instantiate(dentry);
|
|
|
+ }
|
|
|
+ _d_rehash(dentry);
|
|
|
+ if (dir)
|
|
|
+ end_dir_add(dir, n);
|
|
|
+ spin_unlock(&dentry->d_lock);
|
|
|
+ if (inode)
|
|
|
spin_unlock(&inode->i_lock);
|
|
|
- }
|
|
|
- security_d_instantiate(dentry, inode);
|
|
|
- d_rehash(dentry);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -2387,8 +2584,10 @@ static inline void __d_add(struct dentry *dentry, struct inode *inode)
|
|
|
|
|
|
void d_add(struct dentry *entry, struct inode *inode)
|
|
|
{
|
|
|
- if (inode)
|
|
|
+ if (inode) {
|
|
|
+ security_d_instantiate(entry, inode);
|
|
|
spin_lock(&inode->i_lock);
|
|
|
+ }
|
|
|
__d_add(entry, inode);
|
|
|
}
|
|
|
EXPORT_SYMBOL(d_add);
|
|
@@ -2598,6 +2797,8 @@ static void dentry_unlock_for_move(struct dentry *dentry, struct dentry *target)
|
|
|
static void __d_move(struct dentry *dentry, struct dentry *target,
|
|
|
bool exchange)
|
|
|
{
|
|
|
+ struct inode *dir = NULL;
|
|
|
+ unsigned n;
|
|
|
if (!dentry->d_inode)
|
|
|
printk(KERN_WARNING "VFS: moving negative dcache entry\n");
|
|
|
|
|
@@ -2605,6 +2806,11 @@ static void __d_move(struct dentry *dentry, struct dentry *target,
|
|
|
BUG_ON(d_ancestor(target, dentry));
|
|
|
|
|
|
dentry_lock_for_move(dentry, target);
|
|
|
+ if (unlikely(d_in_lookup(target))) {
|
|
|
+ dir = target->d_parent->d_inode;
|
|
|
+ n = start_dir_add(dir);
|
|
|
+ __d_lookup_done(target);
|
|
|
+ }
|
|
|
|
|
|
write_seqcount_begin(&dentry->d_seq);
|
|
|
write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
|
|
@@ -2654,6 +2860,8 @@ static void __d_move(struct dentry *dentry, struct dentry *target,
|
|
|
write_seqcount_end(&target->d_seq);
|
|
|
write_seqcount_end(&dentry->d_seq);
|
|
|
|
|
|
+ if (dir)
|
|
|
+ end_dir_add(dir, n);
|
|
|
dentry_unlock_for_move(dentry, target);
|
|
|
}
|
|
|
|
|
@@ -2724,7 +2932,8 @@ struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
|
|
|
static int __d_unalias(struct inode *inode,
|
|
|
struct dentry *dentry, struct dentry *alias)
|
|
|
{
|
|
|
- struct mutex *m1 = NULL, *m2 = NULL;
|
|
|
+ struct mutex *m1 = NULL;
|
|
|
+ struct rw_semaphore *m2 = NULL;
|
|
|
int ret = -ESTALE;
|
|
|
|
|
|
/* If alias and dentry share a parent, then no extra locks required */
|
|
@@ -2735,15 +2944,15 @@ static int __d_unalias(struct inode *inode,
|
|
|
if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
|
|
|
goto out_err;
|
|
|
m1 = &dentry->d_sb->s_vfs_rename_mutex;
|
|
|
- if (!inode_trylock(alias->d_parent->d_inode))
|
|
|
+ if (!inode_trylock_shared(alias->d_parent->d_inode))
|
|
|
goto out_err;
|
|
|
- m2 = &alias->d_parent->d_inode->i_mutex;
|
|
|
+ m2 = &alias->d_parent->d_inode->i_rwsem;
|
|
|
out_unalias:
|
|
|
__d_move(alias, dentry, false);
|
|
|
ret = 0;
|
|
|
out_err:
|
|
|
if (m2)
|
|
|
- mutex_unlock(m2);
|
|
|
+ up_read(m2);
|
|
|
if (m1)
|
|
|
mutex_unlock(m1);
|
|
|
return ret;
|
|
@@ -2782,6 +2991,7 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
|
|
|
if (!inode)
|
|
|
goto out;
|
|
|
|
|
|
+ security_d_instantiate(dentry, inode);
|
|
|
spin_lock(&inode->i_lock);
|
|
|
if (S_ISDIR(inode->i_mode)) {
|
|
|
struct dentry *new = __d_find_any_alias(inode);
|
|
@@ -2809,7 +3019,6 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
|
|
|
} else {
|
|
|
__d_move(new, dentry, false);
|
|
|
write_sequnlock(&rename_lock);
|
|
|
- security_d_instantiate(new, inode);
|
|
|
}
|
|
|
iput(inode);
|
|
|
return new;
|